From 99e4316a909f360f3e5bf1ab547773a5e5d6b202 Mon Sep 17 00:00:00 2001 From: Teagan King <98482480+TeaganKing@users.noreply.github.com> Date: Thu, 30 Jan 2025 12:17:26 -0700 Subject: [PATCH 01/14] very preliminary updates in helper script for updated cupid workflow --- .../generate_cupid_config_for_cesm_case.py | 36 +++++++++++++++---- 1 file changed, 29 insertions(+), 7 deletions(-) diff --git a/helper_scripts/generate_cupid_config_for_cesm_case.py b/helper_scripts/generate_cupid_config_for_cesm_case.py index d02f75f0..0534f8de 100755 --- a/helper_scripts/generate_cupid_config_for_cesm_case.py +++ b/helper_scripts/generate_cupid_config_for_cesm_case.py @@ -46,7 +46,7 @@ def _parse_args(): return parser.parse_args() -def generate_cupid_config(case_root, cesm_root, cupid_example): +def generate_cupid_config(case_root, cesm_root, cupid_example, cupid_baseline_case=None, cupid_baseline_root=None, cupid_start_year=None, cupid_end_year=None, cupid_base_start_year=None, cupid_base_end_year=None): """ Generate a CUPiD `config.yml` file based on information from a CESM case and a specific CUPiD example configuration (such as 'key metrics'). @@ -108,16 +108,38 @@ def generate_cupid_config(case_root, cesm_root, cupid_example): dout_s_root = cesm_case.get_value("DOUT_S_ROOT") # Additional options we need to get from env_cupid.xml - base_case = "b.e23_alpha17f.BLT1850.ne30_t232.092" - nyears = 1 - start_date = "0001-01-01" - end_date = f"{nyears+1:04d}-01-01" + if cupid_baseline_case: + base_case = cupid_baseline_case + else: + base_case = "b.e23_alpha17f.BLT1850.ne30_t232.092" + if cupid_baseline_root: + base_case_output_dir = cupid_baseline_root + else: + base_case_output_dir = "/glade/campaign/cesm/development/cross-wg/diagnostic_framework/CESM_output_for_testing" + if cupid_start_year: + start_date = cupid_start_year + else: + start_date = "0001-01-01" + if cupid_end_year: + end_date = cupid_end_year + else: + end_date = f"{nyears+1:04d}-01-01" + if cupid_base_start_year: + base_case_start_date = cupid_base_start_year + else: + continue # default for this one? + if cupid_base_end_year: + base_end_date = cupid_base_end_year + else: + base_end_date = f"{base_nyears+1:04d}-01-01" + + # TODO: these should also perhaps be added as environment vars? + nyears = 1 climo_nyears = nyears - base_case_output_dir = "/glade/campaign/cesm/development/cross-wg/diagnostic_framework/CESM_output_for_testing" base_nyears = 100 - base_end_date = f"{base_nyears+1:04d}-01-01" base_climo_nyears = 40 + # -------------------------------------------------------------------------------- with open(os.path.join(cupid_root, "examples", cupid_example, "config.yml")) as f: my_dict = yaml.safe_load(f) From 741e8cb2611384b4354cee7adcb598be54e64e28 Mon Sep 17 00:00:00 2001 From: Teagan King Date: Fri, 7 Feb 2025 13:37:03 -0700 Subject: [PATCH 02/14] a few formatting things --- .../generate_cupid_config_for_cesm_case.py | 29 +++++++++++++------ 1 file changed, 20 insertions(+), 9 deletions(-) diff --git a/helper_scripts/generate_cupid_config_for_cesm_case.py b/helper_scripts/generate_cupid_config_for_cesm_case.py index 0534f8de..5d1a96db 100755 --- a/helper_scripts/generate_cupid_config_for_cesm_case.py +++ b/helper_scripts/generate_cupid_config_for_cesm_case.py @@ -46,7 +46,17 @@ def _parse_args(): return parser.parse_args() -def generate_cupid_config(case_root, cesm_root, cupid_example, cupid_baseline_case=None, cupid_baseline_root=None, cupid_start_year=None, cupid_end_year=None, cupid_base_start_year=None, cupid_base_end_year=None): +def generate_cupid_config( + case_root, + cesm_root, + cupid_example, + cupid_baseline_case=None, + cupid_baseline_root=None, + cupid_start_year=None, + cupid_end_year=None, + cupid_base_start_year=None, + cupid_base_end_year=None, +): """ Generate a CUPiD `config.yml` file based on information from a CESM case and a specific CUPiD example configuration (such as 'key metrics'). @@ -107,6 +117,12 @@ def generate_cupid_config(case_root, cesm_root, cupid_example, cupid_baseline_ca case = cesm_case.get_value("CASE") dout_s_root = cesm_case.get_value("DOUT_S_ROOT") + # TODO: these should also perhaps be added as environment vars? + nyears = 1 + climo_nyears = nyears + base_nyears = 100 + base_climo_nyears = 40 + # Additional options we need to get from env_cupid.xml if cupid_baseline_case: base_case = cupid_baseline_case @@ -125,20 +141,14 @@ def generate_cupid_config(case_root, cesm_root, cupid_example, cupid_baseline_ca else: end_date = f"{nyears+1:04d}-01-01" if cupid_base_start_year: - base_case_start_date = cupid_base_start_year + base_start_date = cupid_base_start_year else: - continue # default for this one? + base_start_date = start_date if cupid_base_end_year: base_end_date = cupid_base_end_year else: base_end_date = f"{base_nyears+1:04d}-01-01" - # TODO: these should also perhaps be added as environment vars? - nyears = 1 - climo_nyears = nyears - base_nyears = 100 - base_climo_nyears = 40 - # -------------------------------------------------------------------------------- with open(os.path.join(cupid_root, "examples", cupid_example, "config.yml")) as f: my_dict = yaml.safe_load(f) @@ -154,6 +164,7 @@ def generate_cupid_config(case_root, cesm_root, cupid_example, cupid_baseline_ca my_dict["global_params"]["end_date"] = end_date my_dict["global_params"]["base_case_name"] = base_case my_dict["global_params"]["base_case_output_dir"] = base_case_output_dir + my_dict["global_params"]["base_start_date"] = base_start_date my_dict["global_params"]["base_end_date"] = base_end_date my_dict["timeseries"]["case_name"] = [case, base_case] From 50ae4a55f86fcad5a87acda3fc705f70e5ec4fa7 Mon Sep 17 00:00:00 2001 From: Teagan King <98482480+TeaganKing@users.noreply.github.com> Date: Mon, 10 Feb 2025 11:06:27 -0700 Subject: [PATCH 03/14] Update generate_cupid_config_for_cesm_case.py to include argument descriptions from xml vars --- .../generate_cupid_config_for_cesm_case.py | 20 +++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/helper_scripts/generate_cupid_config_for_cesm_case.py b/helper_scripts/generate_cupid_config_for_cesm_case.py index 5d1a96db..f361698e 100755 --- a/helper_scripts/generate_cupid_config_for_cesm_case.py +++ b/helper_scripts/generate_cupid_config_for_cesm_case.py @@ -84,6 +84,26 @@ def generate_cupid_config( The name of a CUPiD example (e.g., 'key metrics') to base the configuration file on. Must be a valid subdirectory within the CUPiD examples directory. + Optional Arguments: + ------------------- + cupid_baseline_case : str + The name of the base case. + + cupid_baseline_root : str + The root directory of the base case. + + cupid_start_year : int + The start year of the case being analyzed. + + cupid_end_year : int + The end year of the case being analyzed. + + cupid_base_start_year : int + The start year of the base case. + + cupid_base_end_year : int + The end year of the base case. + Raises: ------- KeyError: From 4939a3d540a898ce67ca38910c3d9f5e682aaad6 Mon Sep 17 00:00:00 2001 From: Teagan King <98482480+TeaganKing@users.noreply.github.com> Date: Mon, 10 Feb 2025 11:08:55 -0700 Subject: [PATCH 04/14] remove whitespace in generate_cupid_config_for_cesm_case.py --- helper_scripts/generate_cupid_config_for_cesm_case.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/helper_scripts/generate_cupid_config_for_cesm_case.py b/helper_scripts/generate_cupid_config_for_cesm_case.py index f361698e..d3c75604 100755 --- a/helper_scripts/generate_cupid_config_for_cesm_case.py +++ b/helper_scripts/generate_cupid_config_for_cesm_case.py @@ -97,7 +97,7 @@ def generate_cupid_config( cupid_end_year : int The end year of the case being analyzed. - + cupid_base_start_year : int The start year of the base case. From 88d6f2c4dc169d95f0ed6307b1bbd650e9f296ca Mon Sep 17 00:00:00 2001 From: Michael Levy Date: Tue, 11 Feb 2025 11:34:56 -0700 Subject: [PATCH 05/14] Add cime_config and update helper scripts cime_config/config_tool.xml is used by CESM to create env_postprocessing.xml. Variables in this XML file are then used to generate the cupid config file for CESM cases --- cime_config/config_tool.xml | 183 ++++++++++++++++++ .../generate_cupid_config_for_cesm_case.py | 111 ++++++----- 2 files changed, 248 insertions(+), 46 deletions(-) create mode 100644 cime_config/config_tool.xml diff --git a/cime_config/config_tool.xml b/cime_config/config_tool.xml new file mode 100644 index 00000000..5fdd6c6f --- /dev/null +++ b/cime_config/config_tool.xml @@ -0,0 +1,183 @@ + + + + + + + + + + char + TRUE,FALSE + TRUE + cupid_config + env_postprocessing.xml + When TRUE, CUPiD will generate time series files from history output + + + + char + TRUE,FALSE + TRUE + cupid_config + env_postprocessing.xml + When TRUE, CUPiD will generate diagnostic plots + + + + char + TRUE,FALSE + TRUE + cupid_config + env_postprocessing.xml + When TRUE, CUPiD will generate web pages from the diagnostic plots + + + + + + char + cupid-analysis + cupid_environments + env_postprocessing.xml + Conda environment to use when running CUPiD notebooks + + + + char + cupid-infrastructure + cupid_environments + env_postprocessing.xml + Conda environment to use when running CUPiD tools + + + + + + char + 0001-01-01 + cupid_analysis + env_postprocessing.xml + First year of output to include in CUPiD analysis + + + + integer + 1 + cupid_analysis + env_postprocessing.xml + Number of years of output to include in CUPiD analysis + + + + char + 0001-01-01 + cupid_analysis + env_postprocessing.xml + First year of baseline output to include in CUPiD analysis + + + + integer + 100 + cupid_analysis + env_postprocessing.xml + Number of years of baseline output to include in CUPiD analysis + + + + char + b.e23_alpha17f.BLT1850.ne30_t232.092 + cupid_analysis + env_postprocessing.xml + Baseline case to use in CUPiD analysis + + + + char + $DOUT_S_ROOT + cupid_analysis + env_postprocessing.xml + Baseline case root to use in CUPiD analysis + + + + char + key_metrics + cupid_analysis + env_postprocessing.xml + Example to use in CUPiD analysis + + + + + + char + TRUE,FALSE + TRUE + cupid_run_components + env_postprocessing.xml + Boolean value that determines whether or not diagnostics for all components are run + + + + char + TRUE,FALSE + FALSE + cupid_run_components + env_postprocessing.xml + Boolean value that determines whether or not ATM diagnostics are run (if CUPID_RUN_ALL is FALSE) + + + + char + TRUE,FALSE + FALSE + cupid_run_components + env_postprocessing.xml + Boolean value that determines whether or not OCN diagnostics are run (if CUPID_RUN_ALL is FALSE) + + + + char + TRUE,FALSE + FALSE + cupid_run_components + env_postprocessing.xml + Boolean value that determines whether or not LND diagnostics are run (if CUPID_RUN_ALL is FALSE) + + + + char + TRUE,FALSE + FALSE + cupid_run_components + env_postprocessing.xml + Boolean value that determines whether or not ICE diagnostics are run (if CUPID_RUN_ALL is FALSE) + + + + char + TRUE,FALSE + FALSE + cupid_run_components + env_postprocessing.xml + Boolean value that determines whether or not ROF diagnostics are run (if CUPID_RUN_ALL is FALSE) + + + + char + TRUE,FALSE + FALSE + cupid_run_components + env_postprocessing.xml + Boolean value that determines whether or not GLC diagnostics are run (if CUPID_RUN_ALL is FALSE) + + + + ========================================= + Configuring CUPiD Postprocessing + ========================================= + + + diff --git a/helper_scripts/generate_cupid_config_for_cesm_case.py b/helper_scripts/generate_cupid_config_for_cesm_case.py index d3c75604..24a7802d 100755 --- a/helper_scripts/generate_cupid_config_for_cesm_case.py +++ b/helper_scripts/generate_cupid_config_for_cesm_case.py @@ -43,6 +43,54 @@ def _parse_args(): help="CESM case directory", ) + parser.add_argument( + "--cupid-baseline-case", + action="store", + default="b.e23_alpha17f.BLT1850.ne30_t232.092", + dest="cupid_baseline_case", + help="Base case name", + ) + + parser.add_argument( + "--cupid-baseline-root", + action="store", + default="/glade/campaign/cesm/development/cross-wg/diagnostic_framework/CESM_output_for_testing", + dest="cupid_baseline_root", + help="Base case root directory", + ) + + parser.add_argument( + "--cupid-start-year", + action="store", + default="0001-01-01", + dest="cupid_start_year", + help="CUPiD case start year", + ) + + parser.add_argument( + "--cupid-end-year", + action="store", + default="0101-01-01", + dest="cupid_end_year", + help="CUPiD case end year", + ) + + parser.add_argument( + "--cupid-base-start-year", + action="store", + default="0001-01-01", + dest="cupid_base_start_year", + help="CUPiD base case start year", + ) + + parser.add_argument( + "--cupid-base-end-year", + action="store", + default="0101-01-01", + dest="cupid_base_end_year", + help="CUPiD base case end year", + ) + return parser.parse_args() @@ -50,12 +98,12 @@ def generate_cupid_config( case_root, cesm_root, cupid_example, - cupid_baseline_case=None, - cupid_baseline_root=None, - cupid_start_year=None, - cupid_end_year=None, - cupid_base_start_year=None, - cupid_base_end_year=None, + cupid_baseline_case, + cupid_baseline_root, + cupid_start_year, + cupid_end_year, + cupid_base_start_year, + cupid_base_end_year, ): """ Generate a CUPiD `config.yml` file based on information from a CESM case and @@ -84,8 +132,6 @@ def generate_cupid_config( The name of a CUPiD example (e.g., 'key metrics') to base the configuration file on. Must be a valid subdirectory within the CUPiD examples directory. - Optional Arguments: - ------------------- cupid_baseline_case : str The name of the base case. @@ -137,38 +183,11 @@ def generate_cupid_config( case = cesm_case.get_value("CASE") dout_s_root = cesm_case.get_value("DOUT_S_ROOT") - # TODO: these should also perhaps be added as environment vars? - nyears = 1 - climo_nyears = nyears - base_nyears = 100 + # TODO: these sea-ice specific vars (and some glc vars) should also be added as environment vars + # See https://github.com/NCAR/CUPiD/issues/189 + climo_nyears = 35 base_climo_nyears = 40 - # Additional options we need to get from env_cupid.xml - if cupid_baseline_case: - base_case = cupid_baseline_case - else: - base_case = "b.e23_alpha17f.BLT1850.ne30_t232.092" - if cupid_baseline_root: - base_case_output_dir = cupid_baseline_root - else: - base_case_output_dir = "/glade/campaign/cesm/development/cross-wg/diagnostic_framework/CESM_output_for_testing" - if cupid_start_year: - start_date = cupid_start_year - else: - start_date = "0001-01-01" - if cupid_end_year: - end_date = cupid_end_year - else: - end_date = f"{nyears+1:04d}-01-01" - if cupid_base_start_year: - base_start_date = cupid_base_start_year - else: - base_start_date = start_date - if cupid_base_end_year: - base_end_date = cupid_base_end_year - else: - base_end_date = f"{base_nyears+1:04d}-01-01" - # -------------------------------------------------------------------------------- with open(os.path.join(cupid_root, "examples", cupid_example, "config.yml")) as f: my_dict = yaml.safe_load(f) @@ -180,20 +199,20 @@ def generate_cupid_config( "nblibrary", ) my_dict["global_params"]["case_name"] = case - my_dict["global_params"]["start_date"] = start_date - my_dict["global_params"]["end_date"] = end_date - my_dict["global_params"]["base_case_name"] = base_case - my_dict["global_params"]["base_case_output_dir"] = base_case_output_dir - my_dict["global_params"]["base_start_date"] = base_start_date - my_dict["global_params"]["base_end_date"] = base_end_date - my_dict["timeseries"]["case_name"] = [case, base_case] + my_dict["global_params"]["start_date"] = cupid_start_year + my_dict["global_params"]["end_date"] = cupid_end_year + my_dict["global_params"]["base_case_name"] = cupid_baseline_case + my_dict["global_params"]["base_case_output_dir"] = cupid_baseline_root + my_dict["global_params"]["base_start_date"] = cupid_base_start_year + my_dict["global_params"]["base_end_date"] = cupid_base_end_year + my_dict["timeseries"]["case_name"] = [case, cupid_baseline_case] for component in my_dict["timeseries"]: if ( isinstance(my_dict["timeseries"][component], dict) and "end_years" in my_dict["timeseries"][component] ): - my_dict["timeseries"][component]["end_years"] = [nyears, base_nyears] + my_dict["timeseries"][component]["end_years"] = [cupid_end_year, cupid_base_end_year] if "link_to_ADF" in my_dict["compute_notebooks"]["atm"]: my_dict["compute_notebooks"]["atm"]["link_to_ADF"]["parameter_groups"]["none"][ "adf_root" From d8ecae98fca6b100c4ddb0fb9ab970887231bc87 Mon Sep 17 00:00:00 2001 From: Teagan King <98482480+TeaganKing@users.noreply.github.com> Date: Tue, 11 Feb 2025 12:10:18 -0700 Subject: [PATCH 06/14] Update config_tool.xml to include ADF xml var --- cime_config/config_tool.xml | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/cime_config/config_tool.xml b/cime_config/config_tool.xml index 5fdd6c6f..81f43e1d 100644 --- a/cime_config/config_tool.xml +++ b/cime_config/config_tool.xml @@ -174,6 +174,15 @@ Boolean value that determines whether or not GLC diagnostics are run (if CUPID_RUN_ALL is FALSE) + + char + TRUE,FALSE + FALSE + cupid_run_components + env_postprocessing.xml + Boolean value that determines whether or not ADF diagnostics are run (if CUPID_RUN_ALL is FALSE) + + ========================================= Configuring CUPiD Postprocessing From e71a14edececb85f78e08f4286cfe6eb059243ae Mon Sep 17 00:00:00 2001 From: Teagan King <98482480+TeaganKing@users.noreply.github.com> Date: Tue, 11 Feb 2025 12:15:29 -0700 Subject: [PATCH 07/14] Update generate_cupid_config_for_cesm_case.py formatting --- helper_scripts/generate_cupid_config_for_cesm_case.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/helper_scripts/generate_cupid_config_for_cesm_case.py b/helper_scripts/generate_cupid_config_for_cesm_case.py index 24a7802d..128d74b9 100755 --- a/helper_scripts/generate_cupid_config_for_cesm_case.py +++ b/helper_scripts/generate_cupid_config_for_cesm_case.py @@ -212,7 +212,10 @@ def generate_cupid_config( isinstance(my_dict["timeseries"][component], dict) and "end_years" in my_dict["timeseries"][component] ): - my_dict["timeseries"][component]["end_years"] = [cupid_end_year, cupid_base_end_year] + my_dict["timeseries"][component]["end_years"] = [ + cupid_end_year, + cupid_base_end_year, + ] if "link_to_ADF" in my_dict["compute_notebooks"]["atm"]: my_dict["compute_notebooks"]["atm"]["link_to_ADF"]["parameter_groups"]["none"][ "adf_root" From 6e85a340871b8a83f7e0682870dd92d4766f1133 Mon Sep 17 00:00:00 2001 From: Michael Levy Date: Wed, 26 Feb 2025 15:49:23 -0700 Subject: [PATCH 08/14] Support base_case_output_dir in seaice notebook --- ...Hemis_seaice_visual_compare_obs_lens.ipynb | 29 +++++++++++++++++-- 1 file changed, 27 insertions(+), 2 deletions(-) diff --git a/nblibrary/ice/Hemis_seaice_visual_compare_obs_lens.ipynb b/nblibrary/ice/Hemis_seaice_visual_compare_obs_lens.ipynb index 896a2423..feb71a0b 100644 --- a/nblibrary/ice/Hemis_seaice_visual_compare_obs_lens.ipynb +++ b/nblibrary/ice/Hemis_seaice_visual_compare_obs_lens.ipynb @@ -59,13 +59,14 @@ "outputs": [], "source": [ "CESM_output_dir = \"\" # \"/glade/campaign/cesm/development/cross-wg/diagnostic_framework/CESM_output_for_testing\"\n", + "base_case_output_dir = None # None => use CESM_output_dir\n", "case_name = \"\" # \"b.e30_beta02.BLT1850.ne30_t232.104\"\n", - "base_case_name = \"\" # \"b.e23_alpha17f.BLT1850.ne30_t232.092\"\n", + "base_case_name = None # \"b.e23_alpha17f.BLT1850.ne30_t232.092\"\n", "\n", "start_date = \"\" # \"0001-01-01\"\n", "end_date = \"\" # \"0101-01-01\"\n", "base_start_date = \"\" # \"0001-01-01\"\n", - "base_end_date = \"\" # \"0101-01-01\"\n", + "base_end_date = None # \"0101-01-01\"\n", "\n", "obs_data_dir = \"\" # \"/glade/campaign/cesm/development/cross-wg/diagnostic_framework/CUPiD_obs_data\"\n", "path_model = \"\" # \"/glade/campaign/cesm/development/cross-wg/diagnostic_framework/CUPiD_model_data/ice/\"\n", @@ -77,6 +78,30 @@ "lc_kwargs = {}" ] }, + { + "cell_type": "code", + "execution_count": null, + "id": "8b1a3efc-9b2b-4342-864d-4d29ef4efa9c", + "metadata": { + "editable": true, + "slideshow": { + "slide_type": "" + }, + "tags": [ + "hide-input" + ] + }, + "outputs": [], + "source": [ + "# Want some base case parameter defaults to equal control case values\n", + "if base_case_name is not None:\n", + " if base_case_output_dir is None:\n", + " base_case_output_dir = CESM_output_dir\n", + "\n", + " if base_end_date is None:\n", + " base_end_date = end_date" + ] + }, { "cell_type": "code", "execution_count": null, From 8f06074adefd5e6242a73e436acab58a9c58b850 Mon Sep 17 00:00:00 2001 From: Michael Levy Date: Wed, 26 Feb 2025 16:19:19 -0700 Subject: [PATCH 09/14] ADF updates 1. Bring in latest ADF commit as external 2. generate_cupid_config_for_cesm_case.py now has --adf-output-root argument that lets users specify a place other than ${CASEROOT} for ADF_output/ Also cleaned up comments about start_years and end_years in timeseries.py --- cupid/timeseries.py | 4 ++-- externals/ADF | 2 +- .../generate_cupid_config_for_cesm_case.py | 24 ++++++++++++++++++- 3 files changed, 26 insertions(+), 4 deletions(-) diff --git a/cupid/timeseries.py b/cupid/timeseries.py index 4f509c2a..dbfe2e8e 100644 --- a/cupid/timeseries.py +++ b/cupid/timeseries.py @@ -65,9 +65,9 @@ def create_time_series( check if time series files already exist - overwrite_ts: list, boolean check if existing time series files will bew overwritten - - start_years: list, str or int + - start_years: list of ints first year for desired range of years - - end_years: list, str or int + - end_years: list of ints last year for desired range of years - height_dim: str name of height dimension for given component, eg 'lev' diff --git a/externals/ADF b/externals/ADF index 8adaef41..7725631d 160000 --- a/externals/ADF +++ b/externals/ADF @@ -1 +1 @@ -Subproject commit 8adaef41dc480cc13937fdf3f712357a13ddeb84 +Subproject commit 7725631dfba06b4025b558e21ef26bcde934eec3 diff --git a/helper_scripts/generate_cupid_config_for_cesm_case.py b/helper_scripts/generate_cupid_config_for_cesm_case.py index 0c2a86a8..1adf35a3 100755 --- a/helper_scripts/generate_cupid_config_for_cesm_case.py +++ b/helper_scripts/generate_cupid_config_for_cesm_case.py @@ -43,6 +43,15 @@ def _parse_args(): help="CESM case directory", ) + # Command line argument location of CESM case directory + parser.add_argument( + "--adf-output-root", + action="store", + dest="adf_output_root", + default=None, + help="Directory where ADF will be run (None => case root)", + ) + parser.add_argument( "--cupid-baseline-case", action="store", @@ -104,6 +113,7 @@ def generate_cupid_config( cupid_end_year, cupid_base_start_year, cupid_base_end_year, + adf_output_root=None, ): """ Generate a CUPiD `config.yml` file based on information from a CESM case and @@ -165,6 +175,10 @@ def generate_cupid_config( sys.path.append(os.path.join(cesm_root, "cime")) from CIME.case import Case + # Is adf_output_root provided? + if adf_output_root is None: + adf_output_root = case_root + # Is cupid_example a valid value? cupid_root = os.path.join(cesm_root, "tools", "CUPiD") cupid_examples = os.path.join(cupid_root, "examples") @@ -198,6 +212,14 @@ def generate_cupid_config( "CUPiD", "nblibrary", ) + if type(cupid_start_year) == str: + cupid_start_year = int(cupid_start_year) + if type(cupid_end_year) == str: + cupid_end_year = int(cupid_end_year) + if type(cupid_base_start_year) == str: + cupid_base_start_year = int(cupid_base_start_year) + if type(cupid_base_end_year) == str: + cupid_base_end_year = int(cupid_base_end_year) my_dict["global_params"]["case_name"] = case my_dict["global_params"]["start_date"] = cupid_start_year my_dict["global_params"]["end_date"] = cupid_end_year @@ -219,7 +241,7 @@ def generate_cupid_config( if "link_to_ADF" in my_dict["compute_notebooks"].get("atm", {}): my_dict["compute_notebooks"]["atm"]["link_to_ADF"]["parameter_groups"]["none"][ "adf_root" - ] = os.path.join(case_root, "ADF_output") + ] = os.path.join(adf_output_root, "ADF_output") if "Greenland_SMB_visual_compare_obs" in my_dict["compute_notebooks"].get( "glc", From 35e362b7fb2e05bd80f98ff7dca92f28315e5b5a Mon Sep 17 00:00:00 2001 From: Michael Levy Date: Wed, 26 Feb 2025 21:10:59 -0700 Subject: [PATCH 10/14] Pass CI --- helper_scripts/generate_cupid_config_for_cesm_case.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/helper_scripts/generate_cupid_config_for_cesm_case.py b/helper_scripts/generate_cupid_config_for_cesm_case.py index 1adf35a3..c59935f8 100755 --- a/helper_scripts/generate_cupid_config_for_cesm_case.py +++ b/helper_scripts/generate_cupid_config_for_cesm_case.py @@ -212,13 +212,13 @@ def generate_cupid_config( "CUPiD", "nblibrary", ) - if type(cupid_start_year) == str: + if isinstance(cupid_start_year, str): cupid_start_year = int(cupid_start_year) - if type(cupid_end_year) == str: + if isinstance(cupid_end_year, str): cupid_end_year = int(cupid_end_year) - if type(cupid_base_start_year) == str: + if isinstance(cupid_base_start_year, str): cupid_base_start_year = int(cupid_base_start_year) - if type(cupid_base_end_year) == str: + if isinstance(cupid_base_end_year, str): cupid_base_end_year = int(cupid_base_end_year) my_dict["global_params"]["case_name"] = case my_dict["global_params"]["start_date"] = cupid_start_year From 250d28c3a08c708250efe8adec3715faa729af99 Mon Sep 17 00:00:00 2001 From: Michael Levy Date: Mon, 3 Mar 2025 14:44:48 -0700 Subject: [PATCH 11/14] Variable name changes in config_tool.xml Keeping with CESM convention, using STARTDATE instead of START_YEAR. This propogates into generate_cupid_config_for_cesm_case.py, which wants to drop the MM-DD portion of the date in favor of saving just the year for timeseries generation (where we assume ENDDATE = YYYY-01-01, so END_YEAR = YYYY - 1) --- cime_config/config_tool.xml | 4 +- .../generate_cupid_config_for_cesm_case.py | 67 +++++++++---------- 2 files changed, 33 insertions(+), 38 deletions(-) diff --git a/cime_config/config_tool.xml b/cime_config/config_tool.xml index 81f43e1d..f27a8c91 100644 --- a/cime_config/config_tool.xml +++ b/cime_config/config_tool.xml @@ -53,7 +53,7 @@ - + char 0001-01-01 cupid_analysis @@ -69,7 +69,7 @@ Number of years of output to include in CUPiD analysis - + char 0001-01-01 cupid_analysis diff --git a/helper_scripts/generate_cupid_config_for_cesm_case.py b/helper_scripts/generate_cupid_config_for_cesm_case.py index c59935f8..3e0d4079 100755 --- a/helper_scripts/generate_cupid_config_for_cesm_case.py +++ b/helper_scripts/generate_cupid_config_for_cesm_case.py @@ -69,35 +69,35 @@ def _parse_args(): ) parser.add_argument( - "--cupid-start-year", + "--cupid-startdate", action="store", default="0001-01-01", - dest="cupid_start_year", - help="CUPiD case start year", + dest="cupid_startdate", + help="CUPiD case start date", ) parser.add_argument( - "--cupid-end-year", + "--cupid-enddate", action="store", default="0101-01-01", - dest="cupid_end_year", - help="CUPiD case end year", + dest="cupid_enddate", + help="CUPiD case end date", ) parser.add_argument( - "--cupid-base-start-year", + "--cupid-base-startdate", action="store", default="0001-01-01", - dest="cupid_base_start_year", - help="CUPiD base case start year", + dest="cupid_base_startdate", + help="CUPiD base case start date", ) parser.add_argument( - "--cupid-base-end-year", + "--cupid-base-enddate", action="store", default="0101-01-01", - dest="cupid_base_end_year", - help="CUPiD base case end year", + dest="cupid_base_enddate", + help="CUPiD base case end date", ) return parser.parse_args() @@ -109,10 +109,10 @@ def generate_cupid_config( cupid_example, cupid_baseline_case, cupid_baseline_root, - cupid_start_year, - cupid_end_year, - cupid_base_start_year, - cupid_base_end_year, + cupid_startdate, + cupid_enddate, + cupid_base_startdate, + cupid_base_enddate, adf_output_root=None, ): """ @@ -148,17 +148,17 @@ def generate_cupid_config( cupid_baseline_root : str The root directory of the base case. - cupid_start_year : int - The start year of the case being analyzed. + cupid_startdate : str + The start date of the case being analyzed ("YYYY-MM-DD"). - cupid_end_year : int - The end year of the case being analyzed. + cupid_enddate : int + The end date of the case being analyzed ("YYYY-MM-DD"). - cupid_base_start_year : int - The start year of the base case. + cupid_base_startdate : str + The start date of the base case ("YYYY-MM-DD"). - cupid_base_end_year : int - The end year of the base case. + cupid_base_enddate : str + The end date of the base case ("YYYY-MM-DD"). Raises: ------- @@ -212,21 +212,13 @@ def generate_cupid_config( "CUPiD", "nblibrary", ) - if isinstance(cupid_start_year, str): - cupid_start_year = int(cupid_start_year) - if isinstance(cupid_end_year, str): - cupid_end_year = int(cupid_end_year) - if isinstance(cupid_base_start_year, str): - cupid_base_start_year = int(cupid_base_start_year) - if isinstance(cupid_base_end_year, str): - cupid_base_end_year = int(cupid_base_end_year) my_dict["global_params"]["case_name"] = case - my_dict["global_params"]["start_date"] = cupid_start_year - my_dict["global_params"]["end_date"] = cupid_end_year + my_dict["global_params"]["start_date"] = cupid_startdate + my_dict["global_params"]["end_date"] = cupid_enddate my_dict["global_params"]["base_case_name"] = cupid_baseline_case my_dict["global_params"]["base_case_output_dir"] = cupid_baseline_root - my_dict["global_params"]["base_start_date"] = cupid_base_start_year - my_dict["global_params"]["base_end_date"] = cupid_base_end_year + my_dict["global_params"]["base_start_date"] = cupid_base_startdate + my_dict["global_params"]["base_end_date"] = cupid_base_enddate my_dict["timeseries"]["case_name"] = [case, cupid_baseline_case] for component in my_dict["timeseries"]: @@ -234,6 +226,9 @@ def generate_cupid_config( isinstance(my_dict["timeseries"][component], dict) and "end_years" in my_dict["timeseries"][component] ): + # Assumption that end_year is YYYY-01-01, so we want end_year to be YYYY-1 + cupid_end_year = int(cupid_enddate.split("-")[0]) - 1 + cupid_base_end_year = int(cupid_base_enddate.split("-")[0]) - 1 my_dict["timeseries"][component]["end_years"] = [ cupid_end_year, cupid_base_end_year, From df6d41e585551c267ad1e8c59bce2958e64e4d0c Mon Sep 17 00:00:00 2001 From: Michael Levy Date: Wed, 5 Mar 2025 15:51:30 -0700 Subject: [PATCH 12/14] Create new script for CESM workflow When you create a new case, CESM will create case.cupid, which in turn will execute the new cesm_postprocessing.sh script -- this means we can update what CUPiD does in the CESM workflow without requiring changes to ccs_config_cesm --- helper_scripts/cesm_postprocessing.sh | 132 ++++++++++++++++++++++++++ 1 file changed, 132 insertions(+) create mode 100755 helper_scripts/cesm_postprocessing.sh diff --git a/helper_scripts/cesm_postprocessing.sh b/helper_scripts/cesm_postprocessing.sh new file mode 100755 index 00000000..b0075ad7 --- /dev/null +++ b/helper_scripts/cesm_postprocessing.sh @@ -0,0 +1,132 @@ +#!/bin/bash -e +# This script is run by the CESM workflow when RUN_POSTPROCESSING=TRUE, +# it is invoked from case.cupid and the expectation is that it is run +# from CASEROOT. +# One possible future development would be to make case.cupid a python script +# and then update this to python as well (and take a CIME Case object as +# an argument) + +# Function to add some number of years to a string that +# is formatted as YYYY-MM-DD and print out the updated +# string in the same format +add_years() { + YEAR=`echo $1 | cut -d '-' -f 1` + MM=`echo $1 | cut -d '-' -f 2` + DD=`echo $1 | cut -d '-' -f 3` + NEW_YEAR=`printf '%04d' "$((YEAR + $2))"`-`printf '%02d' "${MM}"`-`printf '%02d' "${DD}"` + echo ${NEW_YEAR} +} + +# Use cupid-infrastructure environment for running these scripts +# Note: on derecho, the cesmdev module creates a python conflict +# by setting $PYTHONPATH; since this is conda-based we +# want an empty PYTHONPATH environment variable +MACH=`./xmlquery --value MACH` + +# Set variables that come from environment file +CASEROOT=${PWD} +SRCROOT=`./xmlquery --value SRCROOT` +CUPID_EXAMPLE=`./xmlquery --value CUPID_EXAMPLE` +CUPID_GEN_TIMESERIES=`./xmlquery --value CUPID_GEN_TIMESERIES` +CUPID_GEN_DIAGNOSTICS=`./xmlquery --value CUPID_GEN_DIAGNOSTICS` +CUPID_GEN_HTML=`./xmlquery --value CUPID_GEN_HTML` +CUPID_BASELINE_CASE=`./xmlquery --value CUPID_BASELINE_CASE` +CUPID_BASELINE_ROOT=`./xmlquery --value CUPID_BASELINE_ROOT` +CUPID_STARTDATE=`./xmlquery --value CUPID_STARTDATE` +CUPID_NYEARS=`./xmlquery --value CUPID_NYEARS` +CUPID_ENDDATE=`add_years ${CUPID_STARTDATE} ${CUPID_NYEARS}` +CUPID_BASE_STARTDATE=`./xmlquery --value CUPID_BASE_STARTDATE` +CUPID_BASE_NYEARS=`./xmlquery --value CUPID_BASE_NYEARS` +CUPID_BASE_ENDDATE=`add_years ${CUPID_BASE_STARTDATE} ${CUPID_BASE_NYEARS}` +CUPID_RUN_ALL=`./xmlquery --value CUPID_RUN_ALL` +CUPID_RUN_ATM=`./xmlquery --value CUPID_RUN_ATM` +CUPID_RUN_OCN=`./xmlquery --value CUPID_RUN_OCN` +CUPID_RUN_LND=`./xmlquery --value CUPID_RUN_LND` +CUPID_RUN_ICE=`./xmlquery --value CUPID_RUN_ICE` +CUPID_RUN_ROF=`./xmlquery --value CUPID_RUN_ROF` +CUPID_RUN_GLC=`./xmlquery --value CUPID_RUN_GLC` +CUPID_RUN_ADF=`./xmlquery --value CUPID_RUN_ADF` +CUPID_INFRASTRUCTURE_ENV=`./xmlquery --value CUPID_INFRASTRUCTURE_ENV` +CUPID_ANALYSIS_ENV=`./xmlquery --value CUPID_ANALYSIS_ENV` + +# Create directory for running CUPiD +mkdir -p cupid-postprocessing +cd cupid-postprocessing + +# If CUPID_RUN_ALL is TRUE, we don't add any component flags. +# The lack of any component flags tells CUPiD to run all components. +CUPID_FLAG_STRING="" +if [ "${CUPID_RUN_ALL}" == "FALSE" ]; then + if [ "${CUPID_RUN_ATM}" == "TRUE" ]; then + CUPID_FLAG_STRING+=" --atm" + fi + if [ "${CUPID_RUN_OCN}" == "TRUE" ]; then + CUPID_FLAG_STRING+=" --ocn" + fi + if [ "${CUPID_RUN_LND}" == "TRUE" ]; then + CUPID_FLAG_STRING+=" --lnd" + fi + if [ "${CUPID_RUN_ICE}" == "TRUE" ]; then + CUPID_FLAG_STRING+=" --ice" + fi + if [ "${CUPID_RUN_ROF}" == "TRUE" ]; then + CUPID_FLAG_STRING+=" --rof" + fi + if [ "${CUPID_RUN_GLC}" == "TRUE" ]; then + CUPID_FLAG_STRING+=" --glc" + fi + if [ "${CUPID_FLAG_STRING}" == "" ]; then + echo "If CUPID_RUN_ALL is False, user must set at least one component" + exit 1 + fi +fi +if [ "${CUPID_RUN_ALL}" == "TRUE" ]; then + echo "CUPID_RUN_ALL is True, running diagnostics for all components" +fi + +unset PYTHONPATH +conda activate ${CUPID_INFRASTRUCTURE_ENV} + +# 1. Generate CUPiD config file +${SRCROOT}/tools/CUPiD/helper_scripts/generate_cupid_config_for_cesm_case.py \ + --cesm-root ${SRCROOT} \ + --case-root ${CASEROOT} \ + --adf-output-root ${PWD} \ + --cupid-example ${CUPID_EXAMPLE} \ + --cupid-baseline-case ${CUPID_BASELINE_CASE} \ + --cupid-baseline-root ${CUPID_BASELINE_ROOT} \ + --cupid-startdate ${CUPID_STARTDATE} \ + --cupid-enddate ${CUPID_ENDDATE} \ + --cupid-base-startdate ${CUPID_BASE_STARTDATE} \ + --cupid-base-enddate ${CUPID_BASE_ENDDATE} \ + +# 2. Generate ADF config file +if [ "${CUPID_RUN_ADF}" == "TRUE" ]; then + ${SRCROOT}/tools/CUPiD/helper_scripts/generate_adf_config_file.py \ + --cesm-root ${SRCROOT} \ + --cupid-config-loc . \ + --adf-template ${SRCROOT}/tools/CUPiD/externals/ADF/config_amwg_default_plots.yaml \ + --out-file adf_config.yml +fi + +# 3. Generate timeseries files +if [ "${CUPID_GEN_TIMESERIES}" == "TRUE" ]; then + ${SRCROOT}/tools/CUPiD/cupid/run_timeseries.py ${CUPID_FLAG_STRING} +fi + +#4. Run ADF +if [ "${CUPID_RUN_ADF}" == "TRUE" ]; then + conda deactivate + conda activate ${CUPID_ANALYSIS_ENV} + ${SRCROOT}/tools/CUPiD/externals/ADF/run_adf_diag adf_config.yml +fi + +# 5. Run CUPiD and build webpage +conda deactivate +conda activate ${CUPID_INFRASTRUCTURE_ENV} +if [ "${CUPID_GEN_DIAGNOSTICS}" == "TRUE" ]; then + ${SRCROOT}/tools/CUPiD/cupid/run_diagnostics.py ${CUPID_FLAG_STRING} +fi +if [ "${CUPID_GEN_HTML}" == "TRUE" ]; then + ${SRCROOT}/tools/CUPiD/cupid/cupid_webpage.py +fi From 884c8eb285a7d7ec06595a1e4f5d8139b193d875 Mon Sep 17 00:00:00 2001 From: Teagan King <98482480+TeaganKing@users.noreply.github.com> Date: Thu, 6 Mar 2025 09:15:10 -0700 Subject: [PATCH 13/14] Update config_tool.xml comments (minor typo) --- cime_config/config_tool.xml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/cime_config/config_tool.xml b/cime_config/config_tool.xml index f27a8c91..31f7c15c 100644 --- a/cime_config/config_tool.xml +++ b/cime_config/config_tool.xml @@ -4,7 +4,7 @@ - + char @@ -33,7 +33,7 @@ When TRUE, CUPiD will generate web pages from the diagnostic plots - + char @@ -109,7 +109,7 @@ Example to use in CUPiD analysis - + char From ed9c32261b2e6c464ac08b01cab40a13c73938c2 Mon Sep 17 00:00:00 2001 From: Michael Levy Date: Thu, 6 Mar 2025 09:48:22 -0700 Subject: [PATCH 14/14] Address concerns from review 1. Move comment about unsetting python path to where we unset the path 2. correct comment about cupid_enddate datatype Note: I also removed the ./xmlquery MACH call because we weren't using that anywhere (I wonder if we used to only run some commands based on being on derecho, but now it's all controlled by env_postprocessing.xml) --- helper_scripts/cesm_postprocessing.sh | 12 +++++------- .../generate_cupid_config_for_cesm_case.py | 2 +- 2 files changed, 6 insertions(+), 8 deletions(-) diff --git a/helper_scripts/cesm_postprocessing.sh b/helper_scripts/cesm_postprocessing.sh index b0075ad7..13b0c1a8 100755 --- a/helper_scripts/cesm_postprocessing.sh +++ b/helper_scripts/cesm_postprocessing.sh @@ -17,13 +17,7 @@ add_years() { echo ${NEW_YEAR} } -# Use cupid-infrastructure environment for running these scripts -# Note: on derecho, the cesmdev module creates a python conflict -# by setting $PYTHONPATH; since this is conda-based we -# want an empty PYTHONPATH environment variable -MACH=`./xmlquery --value MACH` - -# Set variables that come from environment file +# Set variables that come from environment or CESM XML files CASEROOT=${PWD} SRCROOT=`./xmlquery --value SRCROOT` CUPID_EXAMPLE=`./xmlquery --value CUPID_EXAMPLE` @@ -84,6 +78,10 @@ if [ "${CUPID_RUN_ALL}" == "TRUE" ]; then echo "CUPID_RUN_ALL is True, running diagnostics for all components" fi +# Use cupid-infrastructure environment for running these scripts +# Note: on derecho, the cesmdev module creates a python conflict +# by setting $PYTHONPATH; since this is conda-based we +# want an empty PYTHONPATH environment variable unset PYTHONPATH conda activate ${CUPID_INFRASTRUCTURE_ENV} diff --git a/helper_scripts/generate_cupid_config_for_cesm_case.py b/helper_scripts/generate_cupid_config_for_cesm_case.py index 3e0d4079..816a8bcf 100755 --- a/helper_scripts/generate_cupid_config_for_cesm_case.py +++ b/helper_scripts/generate_cupid_config_for_cesm_case.py @@ -151,7 +151,7 @@ def generate_cupid_config( cupid_startdate : str The start date of the case being analyzed ("YYYY-MM-DD"). - cupid_enddate : int + cupid_enddate : str The end date of the case being analyzed ("YYYY-MM-DD"). cupid_base_startdate : str