diff --git a/cime_config/config_tool.xml b/cime_config/config_tool.xml new file mode 100644 index 00000000..31f7c15c --- /dev/null +++ b/cime_config/config_tool.xml @@ -0,0 +1,192 @@ + + + + + + + + + + char + TRUE,FALSE + TRUE + cupid_config + env_postprocessing.xml + When TRUE, CUPiD will generate time series files from history output + + + + char + TRUE,FALSE + TRUE + cupid_config + env_postprocessing.xml + When TRUE, CUPiD will generate diagnostic plots + + + + char + TRUE,FALSE + TRUE + cupid_config + env_postprocessing.xml + When TRUE, CUPiD will generate web pages from the diagnostic plots + + + + + + char + cupid-analysis + cupid_environments + env_postprocessing.xml + Conda environment to use when running CUPiD notebooks + + + + char + cupid-infrastructure + cupid_environments + env_postprocessing.xml + Conda environment to use when running CUPiD tools + + + + + + char + 0001-01-01 + cupid_analysis + env_postprocessing.xml + First year of output to include in CUPiD analysis + + + + integer + 1 + cupid_analysis + env_postprocessing.xml + Number of years of output to include in CUPiD analysis + + + + char + 0001-01-01 + cupid_analysis + env_postprocessing.xml + First year of baseline output to include in CUPiD analysis + + + + integer + 100 + cupid_analysis + env_postprocessing.xml + Number of years of baseline output to include in CUPiD analysis + + + + char + b.e23_alpha17f.BLT1850.ne30_t232.092 + cupid_analysis + env_postprocessing.xml + Baseline case to use in CUPiD analysis + + + + char + $DOUT_S_ROOT + cupid_analysis + env_postprocessing.xml + Baseline case root to use in CUPiD analysis + + + + char + key_metrics + cupid_analysis + env_postprocessing.xml + Example to use in CUPiD analysis + + + + + + char + TRUE,FALSE + TRUE + cupid_run_components + env_postprocessing.xml + Boolean value that determines whether or not diagnostics for all components are run + + + + char + TRUE,FALSE + FALSE + cupid_run_components + env_postprocessing.xml + Boolean value that determines whether or not ATM diagnostics are run (if CUPID_RUN_ALL is FALSE) + + + + char + TRUE,FALSE + FALSE + cupid_run_components + env_postprocessing.xml + Boolean value that determines whether or not OCN diagnostics are run (if CUPID_RUN_ALL is FALSE) + + + + char + TRUE,FALSE + FALSE + cupid_run_components + env_postprocessing.xml + Boolean value that determines whether or not LND diagnostics are run (if CUPID_RUN_ALL is FALSE) + + + + char + TRUE,FALSE + FALSE + cupid_run_components + env_postprocessing.xml + Boolean value that determines whether or not ICE diagnostics are run (if CUPID_RUN_ALL is FALSE) + + + + char + TRUE,FALSE + FALSE + cupid_run_components + env_postprocessing.xml + Boolean value that determines whether or not ROF diagnostics are run (if CUPID_RUN_ALL is FALSE) + + + + char + TRUE,FALSE + FALSE + cupid_run_components + env_postprocessing.xml + Boolean value that determines whether or not GLC diagnostics are run (if CUPID_RUN_ALL is FALSE) + + + + char + TRUE,FALSE + FALSE + cupid_run_components + env_postprocessing.xml + Boolean value that determines whether or not ADF diagnostics are run (if CUPID_RUN_ALL is FALSE) + + + + ========================================= + Configuring CUPiD Postprocessing + ========================================= + + + diff --git a/cupid/timeseries.py b/cupid/timeseries.py index 4f509c2a..dbfe2e8e 100644 --- a/cupid/timeseries.py +++ b/cupid/timeseries.py @@ -65,9 +65,9 @@ def create_time_series( check if time series files already exist - overwrite_ts: list, boolean check if existing time series files will bew overwritten - - start_years: list, str or int + - start_years: list of ints first year for desired range of years - - end_years: list, str or int + - end_years: list of ints last year for desired range of years - height_dim: str name of height dimension for given component, eg 'lev' diff --git a/externals/ADF b/externals/ADF index 8adaef41..7725631d 160000 --- a/externals/ADF +++ b/externals/ADF @@ -1 +1 @@ -Subproject commit 8adaef41dc480cc13937fdf3f712357a13ddeb84 +Subproject commit 7725631dfba06b4025b558e21ef26bcde934eec3 diff --git a/helper_scripts/cesm_postprocessing.sh b/helper_scripts/cesm_postprocessing.sh new file mode 100755 index 00000000..13b0c1a8 --- /dev/null +++ b/helper_scripts/cesm_postprocessing.sh @@ -0,0 +1,130 @@ +#!/bin/bash -e +# This script is run by the CESM workflow when RUN_POSTPROCESSING=TRUE, +# it is invoked from case.cupid and the expectation is that it is run +# from CASEROOT. +# One possible future development would be to make case.cupid a python script +# and then update this to python as well (and take a CIME Case object as +# an argument) + +# Function to add some number of years to a string that +# is formatted as YYYY-MM-DD and print out the updated +# string in the same format +add_years() { + YEAR=`echo $1 | cut -d '-' -f 1` + MM=`echo $1 | cut -d '-' -f 2` + DD=`echo $1 | cut -d '-' -f 3` + NEW_YEAR=`printf '%04d' "$((YEAR + $2))"`-`printf '%02d' "${MM}"`-`printf '%02d' "${DD}"` + echo ${NEW_YEAR} +} + +# Set variables that come from environment or CESM XML files +CASEROOT=${PWD} +SRCROOT=`./xmlquery --value SRCROOT` +CUPID_EXAMPLE=`./xmlquery --value CUPID_EXAMPLE` +CUPID_GEN_TIMESERIES=`./xmlquery --value CUPID_GEN_TIMESERIES` +CUPID_GEN_DIAGNOSTICS=`./xmlquery --value CUPID_GEN_DIAGNOSTICS` +CUPID_GEN_HTML=`./xmlquery --value CUPID_GEN_HTML` +CUPID_BASELINE_CASE=`./xmlquery --value CUPID_BASELINE_CASE` +CUPID_BASELINE_ROOT=`./xmlquery --value CUPID_BASELINE_ROOT` +CUPID_STARTDATE=`./xmlquery --value CUPID_STARTDATE` +CUPID_NYEARS=`./xmlquery --value CUPID_NYEARS` +CUPID_ENDDATE=`add_years ${CUPID_STARTDATE} ${CUPID_NYEARS}` +CUPID_BASE_STARTDATE=`./xmlquery --value CUPID_BASE_STARTDATE` +CUPID_BASE_NYEARS=`./xmlquery --value CUPID_BASE_NYEARS` +CUPID_BASE_ENDDATE=`add_years ${CUPID_BASE_STARTDATE} ${CUPID_BASE_NYEARS}` +CUPID_RUN_ALL=`./xmlquery --value CUPID_RUN_ALL` +CUPID_RUN_ATM=`./xmlquery --value CUPID_RUN_ATM` +CUPID_RUN_OCN=`./xmlquery --value CUPID_RUN_OCN` +CUPID_RUN_LND=`./xmlquery --value CUPID_RUN_LND` +CUPID_RUN_ICE=`./xmlquery --value CUPID_RUN_ICE` +CUPID_RUN_ROF=`./xmlquery --value CUPID_RUN_ROF` +CUPID_RUN_GLC=`./xmlquery --value CUPID_RUN_GLC` +CUPID_RUN_ADF=`./xmlquery --value CUPID_RUN_ADF` +CUPID_INFRASTRUCTURE_ENV=`./xmlquery --value CUPID_INFRASTRUCTURE_ENV` +CUPID_ANALYSIS_ENV=`./xmlquery --value CUPID_ANALYSIS_ENV` + +# Create directory for running CUPiD +mkdir -p cupid-postprocessing +cd cupid-postprocessing + +# If CUPID_RUN_ALL is TRUE, we don't add any component flags. +# The lack of any component flags tells CUPiD to run all components. +CUPID_FLAG_STRING="" +if [ "${CUPID_RUN_ALL}" == "FALSE" ]; then + if [ "${CUPID_RUN_ATM}" == "TRUE" ]; then + CUPID_FLAG_STRING+=" --atm" + fi + if [ "${CUPID_RUN_OCN}" == "TRUE" ]; then + CUPID_FLAG_STRING+=" --ocn" + fi + if [ "${CUPID_RUN_LND}" == "TRUE" ]; then + CUPID_FLAG_STRING+=" --lnd" + fi + if [ "${CUPID_RUN_ICE}" == "TRUE" ]; then + CUPID_FLAG_STRING+=" --ice" + fi + if [ "${CUPID_RUN_ROF}" == "TRUE" ]; then + CUPID_FLAG_STRING+=" --rof" + fi + if [ "${CUPID_RUN_GLC}" == "TRUE" ]; then + CUPID_FLAG_STRING+=" --glc" + fi + if [ "${CUPID_FLAG_STRING}" == "" ]; then + echo "If CUPID_RUN_ALL is False, user must set at least one component" + exit 1 + fi +fi +if [ "${CUPID_RUN_ALL}" == "TRUE" ]; then + echo "CUPID_RUN_ALL is True, running diagnostics for all components" +fi + +# Use cupid-infrastructure environment for running these scripts +# Note: on derecho, the cesmdev module creates a python conflict +# by setting $PYTHONPATH; since this is conda-based we +# want an empty PYTHONPATH environment variable +unset PYTHONPATH +conda activate ${CUPID_INFRASTRUCTURE_ENV} + +# 1. Generate CUPiD config file +${SRCROOT}/tools/CUPiD/helper_scripts/generate_cupid_config_for_cesm_case.py \ + --cesm-root ${SRCROOT} \ + --case-root ${CASEROOT} \ + --adf-output-root ${PWD} \ + --cupid-example ${CUPID_EXAMPLE} \ + --cupid-baseline-case ${CUPID_BASELINE_CASE} \ + --cupid-baseline-root ${CUPID_BASELINE_ROOT} \ + --cupid-startdate ${CUPID_STARTDATE} \ + --cupid-enddate ${CUPID_ENDDATE} \ + --cupid-base-startdate ${CUPID_BASE_STARTDATE} \ + --cupid-base-enddate ${CUPID_BASE_ENDDATE} \ + +# 2. Generate ADF config file +if [ "${CUPID_RUN_ADF}" == "TRUE" ]; then + ${SRCROOT}/tools/CUPiD/helper_scripts/generate_adf_config_file.py \ + --cesm-root ${SRCROOT} \ + --cupid-config-loc . \ + --adf-template ${SRCROOT}/tools/CUPiD/externals/ADF/config_amwg_default_plots.yaml \ + --out-file adf_config.yml +fi + +# 3. Generate timeseries files +if [ "${CUPID_GEN_TIMESERIES}" == "TRUE" ]; then + ${SRCROOT}/tools/CUPiD/cupid/run_timeseries.py ${CUPID_FLAG_STRING} +fi + +#4. Run ADF +if [ "${CUPID_RUN_ADF}" == "TRUE" ]; then + conda deactivate + conda activate ${CUPID_ANALYSIS_ENV} + ${SRCROOT}/tools/CUPiD/externals/ADF/run_adf_diag adf_config.yml +fi + +# 5. Run CUPiD and build webpage +conda deactivate +conda activate ${CUPID_INFRASTRUCTURE_ENV} +if [ "${CUPID_GEN_DIAGNOSTICS}" == "TRUE" ]; then + ${SRCROOT}/tools/CUPiD/cupid/run_diagnostics.py ${CUPID_FLAG_STRING} +fi +if [ "${CUPID_GEN_HTML}" == "TRUE" ]; then + ${SRCROOT}/tools/CUPiD/cupid/cupid_webpage.py +fi diff --git a/helper_scripts/generate_cupid_config_for_cesm_case.py b/helper_scripts/generate_cupid_config_for_cesm_case.py index 70dbd0ac..816a8bcf 100755 --- a/helper_scripts/generate_cupid_config_for_cesm_case.py +++ b/helper_scripts/generate_cupid_config_for_cesm_case.py @@ -43,10 +43,78 @@ def _parse_args(): help="CESM case directory", ) + # Command line argument location of CESM case directory + parser.add_argument( + "--adf-output-root", + action="store", + dest="adf_output_root", + default=None, + help="Directory where ADF will be run (None => case root)", + ) + + parser.add_argument( + "--cupid-baseline-case", + action="store", + default="b.e23_alpha17f.BLT1850.ne30_t232.092", + dest="cupid_baseline_case", + help="Base case name", + ) + + parser.add_argument( + "--cupid-baseline-root", + action="store", + default="/glade/campaign/cesm/development/cross-wg/diagnostic_framework/CESM_output_for_testing", + dest="cupid_baseline_root", + help="Base case root directory", + ) + + parser.add_argument( + "--cupid-startdate", + action="store", + default="0001-01-01", + dest="cupid_startdate", + help="CUPiD case start date", + ) + + parser.add_argument( + "--cupid-enddate", + action="store", + default="0101-01-01", + dest="cupid_enddate", + help="CUPiD case end date", + ) + + parser.add_argument( + "--cupid-base-startdate", + action="store", + default="0001-01-01", + dest="cupid_base_startdate", + help="CUPiD base case start date", + ) + + parser.add_argument( + "--cupid-base-enddate", + action="store", + default="0101-01-01", + dest="cupid_base_enddate", + help="CUPiD base case end date", + ) + return parser.parse_args() -def generate_cupid_config(case_root, cesm_root, cupid_example): +def generate_cupid_config( + case_root, + cesm_root, + cupid_example, + cupid_baseline_case, + cupid_baseline_root, + cupid_startdate, + cupid_enddate, + cupid_base_startdate, + cupid_base_enddate, + adf_output_root=None, +): """ Generate a CUPiD `config.yml` file based on information from a CESM case and a specific CUPiD example configuration (such as 'key metrics'). @@ -74,6 +142,24 @@ def generate_cupid_config(case_root, cesm_root, cupid_example): The name of a CUPiD example (e.g., 'key metrics') to base the configuration file on. Must be a valid subdirectory within the CUPiD examples directory. + cupid_baseline_case : str + The name of the base case. + + cupid_baseline_root : str + The root directory of the base case. + + cupid_startdate : str + The start date of the case being analyzed ("YYYY-MM-DD"). + + cupid_enddate : str + The end date of the case being analyzed ("YYYY-MM-DD"). + + cupid_base_startdate : str + The start date of the base case ("YYYY-MM-DD"). + + cupid_base_enddate : str + The end date of the base case ("YYYY-MM-DD"). + Raises: ------- KeyError: @@ -89,6 +175,10 @@ def generate_cupid_config(case_root, cesm_root, cupid_example): sys.path.append(os.path.join(cesm_root, "cime")) from CIME.case import Case + # Is adf_output_root provided? + if adf_output_root is None: + adf_output_root = case_root + # Is cupid_example a valid value? cupid_root = os.path.join(cesm_root, "tools", "CUPiD") cupid_examples = os.path.join(cupid_root, "examples") @@ -107,17 +197,12 @@ def generate_cupid_config(case_root, cesm_root, cupid_example): case = cesm_case.get_value("CASE") dout_s_root = cesm_case.get_value("DOUT_S_ROOT") - # Additional options we need to get from env_cupid.xml - base_case = "b.e23_alpha17f.BLT1850.ne30_t232.092" - nyears = 1 - start_date = "0001-01-01" - end_date = f"{nyears+1:04d}-01-01" - climo_nyears = nyears - base_case_output_dir = "/glade/campaign/cesm/development/cross-wg/diagnostic_framework/CESM_output_for_testing" - base_nyears = 100 - base_end_date = f"{base_nyears+1:04d}-01-01" + # TODO: these sea-ice specific vars (and some glc vars) should also be added as environment vars + # See https://github.com/NCAR/CUPiD/issues/189 + climo_nyears = 35 base_climo_nyears = 40 + # -------------------------------------------------------------------------------- with open(os.path.join(cupid_root, "examples", cupid_example, "config.yml")) as f: my_dict = yaml.safe_load(f) @@ -128,23 +213,30 @@ def generate_cupid_config(case_root, cesm_root, cupid_example): "nblibrary", ) my_dict["global_params"]["case_name"] = case - my_dict["global_params"]["start_date"] = start_date - my_dict["global_params"]["end_date"] = end_date - my_dict["global_params"]["base_case_name"] = base_case - my_dict["global_params"]["base_case_output_dir"] = base_case_output_dir - my_dict["global_params"]["base_end_date"] = base_end_date - my_dict["timeseries"]["case_name"] = [case, base_case] + my_dict["global_params"]["start_date"] = cupid_startdate + my_dict["global_params"]["end_date"] = cupid_enddate + my_dict["global_params"]["base_case_name"] = cupid_baseline_case + my_dict["global_params"]["base_case_output_dir"] = cupid_baseline_root + my_dict["global_params"]["base_start_date"] = cupid_base_startdate + my_dict["global_params"]["base_end_date"] = cupid_base_enddate + my_dict["timeseries"]["case_name"] = [case, cupid_baseline_case] for component in my_dict["timeseries"]: if ( isinstance(my_dict["timeseries"][component], dict) and "end_years" in my_dict["timeseries"][component] ): - my_dict["timeseries"][component]["end_years"] = [nyears, base_nyears] + # Assumption that end_year is YYYY-01-01, so we want end_year to be YYYY-1 + cupid_end_year = int(cupid_enddate.split("-")[0]) - 1 + cupid_base_end_year = int(cupid_base_enddate.split("-")[0]) - 1 + my_dict["timeseries"][component]["end_years"] = [ + cupid_end_year, + cupid_base_end_year, + ] if "link_to_ADF" in my_dict["compute_notebooks"].get("atm", {}): my_dict["compute_notebooks"]["atm"]["link_to_ADF"]["parameter_groups"]["none"][ "adf_root" - ] = os.path.join(case_root, "ADF_output") + ] = os.path.join(adf_output_root, "ADF_output") if "Greenland_SMB_visual_compare_obs" in my_dict["compute_notebooks"].get( "glc", diff --git a/nblibrary/ice/Hemis_seaice_visual_compare_obs_lens.ipynb b/nblibrary/ice/Hemis_seaice_visual_compare_obs_lens.ipynb index 896a2423..feb71a0b 100644 --- a/nblibrary/ice/Hemis_seaice_visual_compare_obs_lens.ipynb +++ b/nblibrary/ice/Hemis_seaice_visual_compare_obs_lens.ipynb @@ -59,13 +59,14 @@ "outputs": [], "source": [ "CESM_output_dir = \"\" # \"/glade/campaign/cesm/development/cross-wg/diagnostic_framework/CESM_output_for_testing\"\n", + "base_case_output_dir = None # None => use CESM_output_dir\n", "case_name = \"\" # \"b.e30_beta02.BLT1850.ne30_t232.104\"\n", - "base_case_name = \"\" # \"b.e23_alpha17f.BLT1850.ne30_t232.092\"\n", + "base_case_name = None # \"b.e23_alpha17f.BLT1850.ne30_t232.092\"\n", "\n", "start_date = \"\" # \"0001-01-01\"\n", "end_date = \"\" # \"0101-01-01\"\n", "base_start_date = \"\" # \"0001-01-01\"\n", - "base_end_date = \"\" # \"0101-01-01\"\n", + "base_end_date = None # \"0101-01-01\"\n", "\n", "obs_data_dir = \"\" # \"/glade/campaign/cesm/development/cross-wg/diagnostic_framework/CUPiD_obs_data\"\n", "path_model = \"\" # \"/glade/campaign/cesm/development/cross-wg/diagnostic_framework/CUPiD_model_data/ice/\"\n", @@ -77,6 +78,30 @@ "lc_kwargs = {}" ] }, + { + "cell_type": "code", + "execution_count": null, + "id": "8b1a3efc-9b2b-4342-864d-4d29ef4efa9c", + "metadata": { + "editable": true, + "slideshow": { + "slide_type": "" + }, + "tags": [ + "hide-input" + ] + }, + "outputs": [], + "source": [ + "# Want some base case parameter defaults to equal control case values\n", + "if base_case_name is not None:\n", + " if base_case_output_dir is None:\n", + " base_case_output_dir = CESM_output_dir\n", + "\n", + " if base_end_date is None:\n", + " base_end_date = end_date" + ] + }, { "cell_type": "code", "execution_count": null,