diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index ec82c805aab..415d0b468d2 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -7,7 +7,9 @@ repos: - id: check-xml files: config/ - id: end-of-file-fixer + exclude: doc/ - id: trailing-whitespace + exclude: doc/ - repo: https://github.com/psf/black rev: 22.3.0 hooks: @@ -18,6 +20,6 @@ repos: hooks: - id: pylint args: - - --disable=I,C,R,logging-not-lazy,wildcard-import,unused-wildcard-import,fixme,broad-except,bare-except,eval-used,exec-used,global-statement,logging-format-interpolation,no-name-in-module,arguments-renamed,unspecified-encoding,protected-access,import-error + - --disable=I,C,R,logging-not-lazy,wildcard-import,unused-wildcard-import,fixme,broad-except,bare-except,eval-used,exec-used,global-statement,logging-format-interpolation,no-name-in-module,arguments-renamed,unspecified-encoding,protected-access,import-error,no-member files: CIME exclude: CIME/(tests|Tools|code_checker.py) diff --git a/CIME/SystemTests/system_tests_common.py b/CIME/SystemTests/system_tests_common.py index 5746e0ec601..bbed844d20a 100644 --- a/CIME/SystemTests/system_tests_common.py +++ b/CIME/SystemTests/system_tests_common.py @@ -23,6 +23,7 @@ get_ts_synopsis, generate_baseline, ) +from CIME.config import Config from CIME.provenance import save_test_time, get_test_success from CIME.locked_files import LOCKED_DIR, lock_file, is_locked import CIME.build as build @@ -253,7 +254,9 @@ def run(self, skip_pnl=False): RUN_PHASE, status, comments=("time={:d}".format(int(time_taken))) ) - if get_model() == "e3sm": + config = Config.instance() + + if config.verbose_run_phase: # If run phase worked, remember the time it took in order to improve later walltime ests baseline_root = self._case.get_value("BASELINE_ROOT") if success: @@ -320,7 +323,9 @@ def run(self, skip_pnl=False): ) ) - if get_model() == "cesm" and self._case.get_value("GENERATE_BASELINE"): + if config.baseline_store_teststatus and self._case.get_value( + "GENERATE_BASELINE" + ): baseline_dir = os.path.join( self._case.get_value("BASELINE_ROOT"), self._case.get_value("BASEGEN_CASE"), diff --git a/CIME/SystemTests/system_tests_compare_n.py b/CIME/SystemTests/system_tests_compare_n.py index c0bd054a1e9..b9b53c8c561 100644 --- a/CIME/SystemTests/system_tests_compare_n.py +++ b/CIME/SystemTests/system_tests_compare_n.py @@ -42,7 +42,7 @@ from CIME.XML.standard_module_setup import * from CIME.SystemTests.system_tests_common import SystemTestsCommon from CIME.case import Case -from CIME.utils import get_model +from CIME.config import Config from CIME.test_status import * import shutil, os, glob @@ -182,6 +182,8 @@ def build_phase(self, sharedlib_only=False, model_only=False): # with a with statement in all the API entrances in CIME. subsequent cases were # created via clone, not a with statement, so it's not in a writeable state, # so we need to use a with statement here to put it in a writeable state. + config = Config.instance() + for i in range(1, self.N): with self._cases[i]: if self._separate_builds: @@ -193,7 +195,7 @@ def build_phase(self, sharedlib_only=False, model_only=False): # Although we're doing separate builds, it still makes sense # to share the sharedlibroot area with case1 so we can reuse # pieces of the build from there. - if get_model() != "e3sm": + if config.common_sharedlibroot: # We need to turn off this change for E3SM because it breaks # the MPAS build system ## TODO: ^this logic mimics what's done in SystemTestsCompareTwo diff --git a/CIME/SystemTests/system_tests_compare_two.py b/CIME/SystemTests/system_tests_compare_two.py index c2fa72d1b1b..bdbe47ce6db 100644 --- a/CIME/SystemTests/system_tests_compare_two.py +++ b/CIME/SystemTests/system_tests_compare_two.py @@ -47,7 +47,7 @@ from CIME.XML.standard_module_setup import * from CIME.SystemTests.system_tests_common import SystemTestsCommon from CIME.case import Case -from CIME.utils import get_model +from CIME.config import Config from CIME.test_status import * import shutil, os, glob @@ -217,7 +217,7 @@ def build_phase(self, sharedlib_only=False, model_only=False): # Although we're doing separate builds, it still makes sense # to share the sharedlibroot area with case1 so we can reuse # pieces of the build from there. - if get_model() != "e3sm": + if Config.instance().common_sharedlibroot: # We need to turn off this change for E3SM because it breaks # the MPAS build system self._case2.set_value( diff --git a/CIME/Tools/generate_cylc_workflow.py b/CIME/Tools/generate_cylc_workflow.py index 16bbeec65e8..cac503b342b 100755 --- a/CIME/Tools/generate_cylc_workflow.py +++ b/CIME/Tools/generate_cylc_workflow.py @@ -3,8 +3,12 @@ """ Generates a cylc workflow file for the case. See https://cylc.github.io for details about cylc """ +import os +import sys -from standard_script_setup import * +sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), "..", ".."))) + +from CIME.Tools.standard_script_setup import * from CIME.case import Case from CIME.utils import expect, transform_vars diff --git a/CIME/Tools/save_provenance b/CIME/Tools/save_provenance index dd7046e8cda..7e3506ff50c 100755 --- a/CIME/Tools/save_provenance +++ b/CIME/Tools/save_provenance @@ -7,10 +7,15 @@ This tool provide command-line access to provenance-saving functionality from standard_script_setup import * from CIME.case import Case +from CIME.config import Config from CIME.provenance import * from CIME.utils import get_lids from CIME.get_timing import get_timing +import logging + +logger = logging.getLogger(__name__) + ############################################################################### def parse_command_line(args, description): ############################################################################### @@ -57,31 +62,56 @@ def _main_func(description): ############################################################################### mode, caseroot, lid = parse_command_line(sys.argv, description) with Case(caseroot, read_only=False) as case: + srcroot = case.get_value("SRCROOT") + + customize_path = os.path.join(srcroot, "cime_config", "customize") + + config = Config.load(customize_path) + if mode == "build": expect( False, "Saving build provenance manually is not currently supported " "but it should already always be happening automatically", ) - save_build_provenance(case, lid=lid) + + try: + config.save_build_provenance(case, lid=lid) + except AttributeError: + logger.debug("Could not save build provenance, no handler found") elif mode == "prerun": expect(lid is not None, "You must provide LID for prerun mode") - save_prerun_provenance(case, lid=lid) + + try: + config.save_prerun_provenance(case, lid=lid) + except AttributeError: + logger.debug("Could not save prerun provenance, no handler found") elif mode == "postrun": expect(lid is None, "Please allow me to autodetect LID") + model = case.get_value("MODEL") caseid = case.get_value("CASE") case.set_value("SAVE_TIMING", True) lids = get_lids(case) + for lid in lids: # call get_timing if needed expected_timing_file = os.path.join( caseroot, "timing", "{}_timing.{}.{}.gz".format(model, caseid, lid) ) + if not os.path.exists(expected_timing_file): get_timing(case, lid) - save_prerun_provenance(case, lid=lid) - save_postrun_provenance(case, lid=lid) + + try: + config.save_prerun_provenance(case, lid=lid) + except AttributeError: + logger.debug("Could not save prerun provenance, no handler found") + + try: + config.save_postrun_provenance(case, lid=lid) + except AttributeError: + logger.debug("Could not save postrun provenance, no handler found") else: expect(False, "Unhandled mode '{}'".format(mode)) diff --git a/CIME/Tools/testreporter.py b/CIME/Tools/testreporter.py index f42d94b4e7a..a76c113ac09 100755 --- a/CIME/Tools/testreporter.py +++ b/CIME/Tools/testreporter.py @@ -3,8 +3,12 @@ """ Simple script to populate CESM test database with test results. """ +import os +import sys -from standard_script_setup import * +sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), "..", ".."))) + +from CIME.Tools.standard_script_setup import * from CIME.XML.env_build import EnvBuild from CIME.XML.env_case import EnvCase diff --git a/CIME/XML/archive.py b/CIME/XML/archive.py index eb7b64c7634..e9d13eae686 100644 --- a/CIME/XML/archive.py +++ b/CIME/XML/archive.py @@ -3,9 +3,9 @@ """ from CIME.XML.standard_module_setup import * +from CIME.config import Config from CIME.XML.archive_base import ArchiveBase from CIME.XML.files import Files -from CIME.utils import expect, get_model from copy import deepcopy logger = logging.getLogger(__name__) @@ -28,12 +28,14 @@ def setup(self, env_archive, components, files=None): components_node = env_archive.make_child( "components", attributes={"version": "2.0"} ) + arch_components = deepcopy(components) - model = get_model() - if "drv" not in arch_components and model != "ufs": - arch_components.append("drv") - if "dart" not in arch_components and model == "cesm": - arch_components.append("dart") + + config = Config.instance() + + for comp in config.additional_archive_components: + if comp not in arch_components: + arch_components.append(comp) for comp in arch_components: infile = files.get_value("ARCHIVE_SPEC_FILE", {"component": comp}) diff --git a/CIME/XML/test_reporter.py b/CIME/XML/test_reporter.py index fc65defda09..3ef87c957a4 100644 --- a/CIME/XML/test_reporter.py +++ b/CIME/XML/test_reporter.py @@ -7,7 +7,6 @@ import urllib.request from CIME.XML.standard_module_setup import * from CIME.XML.generic_xml import GenericXML -from CIME.utils import expect, get_model import ssl # pylint: disable=protected-access @@ -19,11 +18,6 @@ def __init__(self): """ initialize an object """ - - expect( - get_model() == "cesm", - "testreport is only meant to populate the CESM test database.", - ) self.root = None GenericXML.__init__( diff --git a/CIME/bless_test_results.py b/CIME/bless_test_results.py index 1c0ce9018d2..3fe8c710213 100644 --- a/CIME/bless_test_results.py +++ b/CIME/bless_test_results.py @@ -3,10 +3,10 @@ from CIME.utils import ( run_cmd, get_scripts_root, - get_model, EnvironmentContext, parse_test_name, ) +from CIME.config import Config from CIME.test_status import * from CIME.hist_utils import generate_baseline, compare_baseline from CIME.case import Case @@ -36,10 +36,11 @@ def bless_namelists( if not report_only and ( force or input("Update namelists (y/n)? ").upper() in ["Y", "YES"] ): + config = Config.instance() create_test_gen_args = " -g {} ".format( baseline_name - if get_model() == "cesm" + if config.create_test_flag_mode == "cesm" else " -g -b {} ".format(baseline_name) ) if new_test_root is not None: diff --git a/CIME/build.py b/CIME/build.py index 191df132f76..85381b2d9b2 100644 --- a/CIME/build.py +++ b/CIME/build.py @@ -19,12 +19,14 @@ get_logging_options, import_from_file, ) -from CIME.provenance import save_build_provenance as save_build_provenance_sub +from CIME.config import Config from CIME.locked_files import lock_file, unlock_file from CIME.XML.files import Files logger = logging.getLogger(__name__) +config = Config.instance() + _CMD_ARGS_FOR_BUILD = ( "CASEROOT", "CASETOOLS", @@ -283,7 +285,7 @@ def uses_kokkos(case): cam_target = case.get_value("CAM_TARGET") # atm_comp = case.get_value("COMP_ATM") # scream does not use the shared kokkoslib for now - return get_model() == "e3sm" and cam_target in ( + return config.use_kokkos and cam_target in ( "preqx_kokkos", "theta-l", "theta-l_kokkos", @@ -323,7 +325,7 @@ def _build_model( # special case for clm # clm 4_5 and newer is a shared (as in sharedlibs, shared by all tests) library # (but not in E3SM) and should be built in build_libraries - if get_model() != "e3sm" and comp == "clm": + if config.shared_clm_component and comp == "clm": continue else: logger.info(" - Building {} Library ".format(model)) @@ -379,7 +381,7 @@ def _build_model( file_build = os.path.join(exeroot, "{}.bldlog.{}".format(cime_model, lid)) ufs_driver = os.environ.get("UFS_DRIVER") - if cime_model == "ufs" and ufs_driver == "nems": + if config.ufs_alternative_config and ufs_driver == "nems": config_dir = os.path.join( cimeroot, os.pardir, "src", "model", "NEMS", "cime", "cime_config" ) @@ -813,7 +815,7 @@ def _build_libraries( logger.warning(line) # clm not a shared lib for E3SM - if get_model() != "e3sm" and (buildlist is None or "lnd" in buildlist): + if config.shared_clm_component and (buildlist is None or "lnd" in buildlist): comp_lnd = case.get_value("COMP_LND") if comp_lnd == "clm": logging.info(" - Building clm library ") @@ -888,7 +890,7 @@ def _build_model_thread( libroot=libroot, bldroot=bldroot, ) - if get_model() != "ufs": + if config.enable_smp: compile_cmd = "SMP={} {}".format(stringify_bool(smp), compile_cmd) if is_python_executable(cmd): @@ -1104,6 +1106,7 @@ def _case_build_impl( os.environ["BUILD_THREADED"] = stringify_bool(build_threaded) cime_model = get_model() + # TODO need some other method than a flag. if cime_model == "e3sm" and mach == "titan" and compiler == "pgiacc": case.set_value("CAM_TARGET", "preqx_acc") @@ -1175,7 +1178,7 @@ def _case_build_impl( ) if not sharedlib_only: - if get_model() == "e3sm": + if config.build_model_use_cmake: logs.extend( _build_model_cmake( exeroot, @@ -1242,7 +1245,10 @@ def post_build(case, logs, build_complete=False, save_build_provenance=True): os.environ["LID"] if "LID" in os.environ else get_timestamp("%y%m%d-%H%M%S") ) if save_build_provenance: - save_build_provenance_sub(case, lid=lid) + try: + Config.instance().save_build_provenance(case, lid=lid) + except AttributeError: + logger.debug("No handler for save_build_provenance was found") # Set XML to indicate build complete case.set_value("BUILD_COMPLETE", True) case.set_value("BUILD_STATUS", 0) diff --git a/CIME/build_scripts/buildlib.mct b/CIME/build_scripts/buildlib.mct index 855e82e6416..446020839e4 100755 --- a/CIME/build_scripts/buildlib.mct +++ b/CIME/build_scripts/buildlib.mct @@ -5,8 +5,9 @@ _CIMEROOT = os.getenv("CIMEROOT") sys.path.append(os.path.join(_CIMEROOT, "CIME", "Tools")) from standard_script_setup import * +from CIME.config import Config from CIME import utils -from CIME.utils import copyifnewer, run_bld_cmd_ensure_logging, get_model, expect +from CIME.utils import copyifnewer, run_bld_cmd_ensure_logging, expect from CIME.case import Case from CIME.build import get_standard_makefile_args import glob @@ -59,16 +60,17 @@ def buildlib(bldroot, installpath, case): ) srcroot = case.get_value("SRCROOT") - if get_model() == "cesm": - mct_dir = os.path.join(srcroot, "libraries", "mct") - else: - mct_dir = os.path.join(srcroot, "externals", "mct") + customize_path = os.path.join(srcroot, "cime_config", "customize") + + config = Config.load(customize_path) + + mct_path = config.mct_path.format(srcroot=srcroot) for _dir in ("mct", "mpeu"): if not os.path.isdir(os.path.join(bldroot, _dir)): os.makedirs(os.path.join(bldroot, _dir)) copyifnewer( - os.path.join(mct_dir, _dir, "Makefile"), + os.path.join(mct_path, _dir, "Makefile"), os.path.join(bldroot, _dir, "Makefile"), ) @@ -84,10 +86,10 @@ def buildlib(bldroot, installpath, case): run_bld_cmd_ensure_logging(cmd, logger) # Now we run the mct make command - gmake_opts = "-f {} ".format(os.path.join(mct_dir, "Makefile")) + gmake_opts = "-f {} ".format(os.path.join(mct_path, "Makefile")) gmake_opts += " -C {} ".format(bldroot) gmake_opts += " -j {} ".format(case.get_value("GMAKE_J")) - gmake_opts += " SRCDIR={} ".format(os.path.join(mct_dir)) + gmake_opts += " SRCDIR={} ".format(os.path.join(mct_path)) cmd = "{} {}".format(gmake_cmd, gmake_opts) run_bld_cmd_ensure_logging(cmd, logger) diff --git a/CIME/build_scripts/buildlib.mpi-serial b/CIME/build_scripts/buildlib.mpi-serial index 0fa354aa27c..83ad88367fd 100755 --- a/CIME/build_scripts/buildlib.mpi-serial +++ b/CIME/build_scripts/buildlib.mpi-serial @@ -3,7 +3,8 @@ import os, sys, logging, argparse from standard_script_setup import * from CIME import utils -from CIME.utils import copyifnewer, run_bld_cmd_ensure_logging, get_model +from CIME.config import Config +from CIME.utils import copyifnewer, run_bld_cmd_ensure_logging from CIME.case import Case from CIME.build import get_standard_makefile_args import glob @@ -50,12 +51,13 @@ def buildlib(bldroot, installpath, case): caseroot = case.get_value("CASEROOT") srcroot = case.get_value("SRCROOT") - if get_model() == "cesm": - mct_dir = os.path.join(srcroot, "libraries", "mct") - else: - mct_dir = os.path.join(srcroot, "externals", "mct") + customize_path = os.path.join(srcroot, "cime_config", "customize") - for _file in glob.iglob(os.path.join(mct_dir, "mpi-serial", "*.h")): + config = Config.load(customize_path) + + mct_path = config.mct_path.format(srcroot=srcroot) + + for _file in glob.iglob(os.path.join(mct_path, "mpi-serial", "*.h")): copyifnewer(_file, os.path.join(bldroot, os.path.basename(_file))) gmake_opts = "-f {} ".format(os.path.join(caseroot, "Tools", "Makefile")) @@ -72,10 +74,10 @@ def buildlib(bldroot, installpath, case): run_bld_cmd_ensure_logging(cmd, logger) # Now we run the mpi-serial make command - gmake_opts = "-f {} ".format(os.path.join(mct_dir, "mpi-serial", "Makefile")) + gmake_opts = "-f {} ".format(os.path.join(mct_path, "mpi-serial", "Makefile")) gmake_opts += " -C {} ".format(bldroot) gmake_opts += " -j {} ".format(case.get_value("GMAKE_J")) - gmake_opts += " SRCDIR={} ".format(os.path.join(mct_dir)) + gmake_opts += " SRCDIR={} ".format(os.path.join(mct_path)) cmd = "{} {}".format(gmake_cmd, gmake_opts) run_bld_cmd_ensure_logging(cmd, logger) diff --git a/CIME/buildlib.py b/CIME/buildlib.py index 510ef301125..b1152f8923a 100644 --- a/CIME/buildlib.py +++ b/CIME/buildlib.py @@ -7,9 +7,9 @@ from CIME.utils import ( parse_args_and_handle_standard_logging_options, setup_standard_logging_options, - get_model, safe_copy, ) +from CIME.config import Config from CIME.build import get_standard_makefile_args from CIME.XML.files import Files @@ -78,8 +78,10 @@ def build_cime_component_lib(case, compname, libroot, bldroot): with open(os.path.join(confdir, "CIME_cppdefs"), "w") as out: out.write("") + config = Config.instance() + # Build the component - if get_model() != "e3sm": + if config.build_cime_component_lib: safe_copy(os.path.join(confdir, "Filepath"), bldroot) if os.path.exists(os.path.join(confdir, "CIME_cppdefs")): safe_copy(os.path.join(confdir, "CIME_cppdefs"), bldroot) diff --git a/CIME/case/case.py b/CIME/case/case.py index 70849239356..01bf84930c6 100644 --- a/CIME/case/case.py +++ b/CIME/case/case.py @@ -12,6 +12,7 @@ # pylint: disable=import-error,redefined-builtin from CIME import utils +from CIME.config import Config from CIME.utils import expect, get_cime_root, append_status from CIME.utils import convert_to_type, get_model, set_model from CIME.utils import get_project, get_charge_account, check_name @@ -43,6 +44,8 @@ logger = logging.getLogger(__name__) +config = Config.instance() + class Case(object): """ @@ -129,6 +132,12 @@ def __init__(self, case_root=None, read_only=True, record=False, non_local=False if srcroot is not None: utils.GLOBAL["SRCROOT"] = srcroot + # srcroot may not be known yet, in the instance of creating + # a new case + customize_path = os.path.join(srcroot, "cime_config", "customize") + + config.load(customize_path) + if record: self.record_cmd() @@ -634,18 +643,18 @@ def _set_compset(self, compset_name, files): ) self.set_lookup_value("COMP_INTERFACE", self._comp_interface) - if self._cime_model == "ufs": - ufs_driver = os.environ.get("UFS_DRIVER") - attribute = None - if ufs_driver: - attribute = {"component": "nems"} - comp_root_dir_cpl = files.get_value( - "COMP_ROOT_DIR_CPL", attribute=attribute - ) - elif self._cime_model == "cesm": - comp_root_dir_cpl = files.get_value("COMP_ROOT_DIR_CPL") + if config.set_comp_root_dir_cpl: + if config.use_nems_comp_root_dir: + ufs_driver = os.environ.get("UFS_DRIVER") + attribute = None + if ufs_driver: + attribute = {"component": "nems"} + comp_root_dir_cpl = files.get_value( + "COMP_ROOT_DIR_CPL", attribute=attribute + ) + else: + comp_root_dir_cpl = files.get_value("COMP_ROOT_DIR_CPL") - if self._cime_model in ("cesm", "ufs"): self.set_lookup_value("COMP_ROOT_DIR_CPL", comp_root_dir_cpl) # Loop through all of the files listed in COMPSETS_SPEC_FILE and find the file @@ -1489,7 +1498,7 @@ def configure( # Turn on short term archiving as cesm default setting model = get_model() self.set_model_version(model) - if model == "cesm" and not test: + if config.default_short_term_archiving and not test: self.set_value("DOUT_S", True) self.set_value("TIMER_LEVEL", 4) @@ -1680,7 +1689,7 @@ def _create_caseroot_tools(self): ) ) - if get_model() == "e3sm": + if config.copy_e3sm_tools: if os.path.exists(os.path.join(machines_dir, "syslog.{}".format(machine))): safe_copy( os.path.join(machines_dir, "syslog.{}".format(machine)), @@ -1695,7 +1704,7 @@ def _create_caseroot_tools(self): safe_copy(os.path.join(toolsdir, "e3sm_compile_wrap.py"), casetools) # add archive_metadata to the CASEROOT but only for CESM - if get_model() == "cesm": + if config.copy_cesm_tools: try: exefile = os.path.join(toolsdir, "archive_metadata") destfile = os.path.join(self._caseroot, os.path.basename(exefile)) @@ -1745,7 +1754,7 @@ def _create_caseroot_sourcemods(self): with open(readme_file, "w") as fd: fd.write(readme_message.format(component=component)) - if get_model() == "cesm": + if config.copy_cism_source_mods: # Note: this is CESM specific, given that we are referencing cism explitly if "cism" in components: directory = os.path.join( @@ -2024,7 +2033,7 @@ def get_mpirun_cmd(self, job=None, allow_unresolved_envvars=True, overrides=None mpi_arg_string += " : " ngpus_per_node = self.get_value("NGPUS_PER_NODE") - if ngpus_per_node and ngpus_per_node > 0 and self._cime_model != "e3sm": + if ngpus_per_node and ngpus_per_node > 0 and config.gpus_use_set_device_rank: # 1. this setting is tested on Casper only and may not work on other machines # 2. need to be revisited in the future for a more adaptable implementation rundir = self.get_value("RUNDIR") @@ -2306,6 +2315,10 @@ def create( # Propagate to `GenericXML` to resolve $SRCROOT utils.GLOBAL["SRCROOT"] = srcroot + customize_path = os.path.join(srcroot, "cime_config", "customize") + + config.load(customize_path) + # If any of the top level user_mods_dirs contain a config_grids.xml file and # gridfile was not set on the command line, use it. However, if there are # multiple user_mods_dirs, it is an error for more than one of them to contain diff --git a/CIME/case/case_run.py b/CIME/case/case_run.py index 23f6a4968f4..b026262cc55 100644 --- a/CIME/case/case_run.py +++ b/CIME/case/case_run.py @@ -2,11 +2,11 @@ case_run is a member of Class Case '""" from CIME.XML.standard_module_setup import * +from CIME.config import Config from CIME.utils import gzip_existing_file, new_lid, run_and_log_case_status from CIME.utils import run_sub_or_cmd, append_status, safe_copy, model_log, CIMEError from CIME.utils import get_model, batch_jobid from CIME.get_timing import get_timing -from CIME.provenance import save_prerun_provenance, save_postrun_provenance import shutil, time, sys, os, glob @@ -144,7 +144,10 @@ def _run_model_impl(case, lid, skip_pnl=False, da_cycle=0): time.strftime("%Y-%m-%d %H:%M:%S") ), ) - save_prerun_provenance(case) + try: + Config.instance().save_prerun_provenance(case) + except AttributeError: + logger.debug("No hook for saving prerun provenance was executed") model_log( "e3sm", logger, @@ -536,7 +539,10 @@ def case_run(self, skip_pnl=False, set_continue_run=False, submit_resubmits=Fals time.strftime("%Y-%m-%d %H:%M:%S") ), ) - save_postrun_provenance(self) + try: + Config.instance().save_postrun_provenance(self, lid) + except AttributeError: + logger.debug("No hook for saving postrun provenance was executed") model_log( "e3sm", logger, diff --git a/CIME/case/case_setup.py b/CIME/case/case_setup.py index 4beb989675f..65de51eed9f 100644 --- a/CIME/case/case_setup.py +++ b/CIME/case/case_setup.py @@ -6,7 +6,7 @@ import os from CIME.XML.standard_module_setup import * - +from CIME.config import Config from CIME.XML.machines import Machines from CIME.BuildTools.configure import ( configure, @@ -15,7 +15,6 @@ ) from CIME.utils import ( run_and_log_case_status, - get_model, get_batch_script_for_job, safe_copy, file_contains_python_function, @@ -408,7 +407,10 @@ def _case_setup_impl( # create batch files env_batch.make_all_batch_files(case) - if get_model() == "e3sm" and not case.get_value("TEST"): + + if Config.instance().make_case_run_batch_script and not case.get_value( + "TEST" + ): input_batch_script = os.path.join( case.get_value("MACHDIR"), "template.case.run.sh" ) @@ -451,7 +453,9 @@ def _case_setup_impl( ) # Some tests need namelists created here (ERP) - so do this if we are in test mode - if (test_mode or get_model() == "e3sm") and not non_local: + if ( + test_mode or Config.instance().case_setup_generate_namelist + ) and not non_local: logger.info("Generating component namelists as part of setup") case.create_namelists() diff --git a/CIME/config.py b/CIME/config.py new file mode 100644 index 00000000000..5bbc86a2f1d --- /dev/null +++ b/CIME/config.py @@ -0,0 +1,305 @@ +import sys +import glob +import logging +import importlib + +from CIME import utils + +logger = logging.getLogger(__name__) + + +class Config: + def __new__(cls): + if not hasattr(cls, "_instance"): + cls._instance = super(Config, cls).__new__(cls) + + return cls._instance + + def __init__(self): + if getattr(self, "_loaded", False): + return + + self._attribute_config = {} + + self._set_attribute( + "additional_archive_components", + ("drv", "dart"), + desc="Additional components to archive.", + ) + self._set_attribute( + "verbose_run_phase", + False, + desc="If set to `True` then after a SystemTests successful run phase the elapsed time is recorded to BASELINE_ROOT, on a failure the test is checked against the previous run and potential breaking merges are listed in the testlog.", + ) + self._set_attribute( + "baseline_store_teststatus", + True, + desc="If set to `True` and GENERATE_BASELINE is set then a teststatus.log is created in the case's baseline.", + ) + self._set_attribute( + "common_sharedlibroot", + True, + desc="If set to `True` then SHAREDLIBROOT is set for the case and SystemTests will only build the shared libs once.", + ) + self._set_attribute( + "create_test_flag_mode", + "cesm", + desc="Sets the flag mode for the `create_test` script. When set to `cesm`, the `-c` flag will compare baselines against a give directory.", + ) + self._set_attribute( + "use_kokkos", + False, + desc="If set to `True` and CAM_TARGET is `preqx_kokkos`, `theta-l` or `theta-l_kokkos` then kokkos is built with the shared libs.", + ) + self._set_attribute( + "shared_clm_component", + True, + desc="If set to `True` and then the `clm` land component is built as a shared lib.", + ) + self._set_attribute( + "ufs_alternative_config", + False, + desc="If set to `True` and UFS_DRIVER is set to `nems` then model config dir is set to `$CIMEROOT/../src/model/NEMS/cime/cime_config`.", + ) + self._set_attribute( + "enable_smp", + True, + desc="If set to `True` then `SMP=` is added to model compile command.", + ) + self._set_attribute( + "build_model_use_cmake", + False, + desc="If set to `True` the model is built using using CMake otherwise Make is used.", + ) + self._set_attribute( + "build_cime_component_lib", + True, + desc="If set to `True` then `Filepath`, `CIME_cppdefs` and `CCSM_cppdefs` directories are copied from CASEBUILD directory to BUILDROOT in order to build CIME's internal components.", + ) + self._set_attribute( + "default_short_term_archiving", + True, + desc="If set to `True` and the case is not a test then DOUT_S is set to True and TIMER_LEVEL is set to 4.", + ) + # TODO combine copy_e3sm_tools and copy_cesm_tools into a single variable + self._set_attribute( + "copy_e3sm_tools", + False, + desc="If set to `True` then E3SM specific tools are copied into the case directory.", + ) + self._set_attribute( + "copy_cesm_tools", + True, + desc="If set to `True` then CESM specific tools are copied into the case directory.", + ) + self._set_attribute( + "copy_cism_source_mods", + True, + desc="If set to `True` then `$CASEROOT/SourceMods/src.cism/source_cism` is created and a README is written to directory.", + ) + self._set_attribute( + "make_case_run_batch_script", + False, + desc="If set to `True` and case is not a test then `case.run.sh` is created in case directory from `$MACHDIR/template.case.run.sh`.", + ) + self._set_attribute( + "case_setup_generate_namelist", + False, + desc="If set to `True` and case is a test then namelists are created during `case.setup`.", + ) + self._set_attribute( + "create_bless_log", + False, + desc="If set to `True` and comparing test to baselines the most recent bless is added to comments.", + ) + self._set_attribute( + "allow_unsupported", + True, + desc="If set to `True` then unsupported compsets and resolutions are allowed.", + ) + # set for ufs + self._set_attribute( + "check_machine_name_from_test_name", + True, + desc="If set to `True` then the TestScheduler will use testlists to parse for a list of tests.", + ) + self._set_attribute( + "sort_tests", + False, + desc="If set to `True` then the TestScheduler will sort tests by runtime.", + ) + self._set_attribute( + "calculate_mode_build_cost", + False, + desc="If set to `True` then the TestScheduler will set the number of processors for building the model to min(16, (($GMAKE_J * 2) / 3) + 1) otherwise it's set to 4.", + ) + self._set_attribute( + "share_exes", + False, + desc="If set to `True` then the TestScheduler will share exes between tests.", + ) + + self._set_attribute( + "serialize_sharedlib_builds", + True, + desc="If set to `True` then the TestScheduler will use `proc_pool + 1` processors to build shared libraries otherwise a single processor is used.", + ) + + self._set_attribute( + "use_testreporter_template", + True, + desc="If set to `True` then the TestScheduler will create `testreporter` in $CIME_OUTPUT_ROOT.", + ) + + self._set_attribute( + "check_invalid_args", + True, + desc="If set to `True` then script arguments are checked for being valid.", + ) + self._set_attribute( + "test_mode", + "cesm", + desc="Sets the testing mode, this changes various configuration for CIME's unit and system tests.", + ) + self._set_attribute( + "xml_component_key", + "COMP_ROOT_DIR_{}", + desc="The string template used as the key to query the XML system to find a components root directory e.g. the template `COMP_ROOT_DIR_{}` and component `LND` becomes `COMP_ROOT_DIR_LND`.", + ) + self._set_attribute( + "set_comp_root_dir_cpl", + True, + desc="If set to `True` then COMP_ROOT_DIR_CPL is set for the case.", + ) + self._set_attribute( + "use_nems_comp_root_dir", + False, + desc="If set to `True` then COMP_ROOT_DIR_CPL is set using UFS_DRIVER if defined.", + ) + self._set_attribute( + "gpus_use_set_device_rank", + True, + desc="If set to `True` and NGPUS_PER_NODE > 0 then `$RUNDIR/set_device_rank.sh` is appended when the MPI run command is generated.", + ) + self._set_attribute( + "test_custom_project_machine", + "melvin", + desc="Sets the machine name to use when testing a machine with no PROJECT.", + ) + self._set_attribute( + "driver_default", "nuopc", desc="Sets the default driver for the model." + ) + self._set_attribute( + "driver_choices", + ("mct", "nuopc"), + desc="Sets the available driver choices for the model.", + ) + self._set_attribute( + "mct_path", + "{srcroot}/libraries/mct", + desc="Sets the path to the mct library.", + ) + + @classmethod + def instance(cls): + """Access singleton. + + Explicit way to access singleton, same as calling constructor. + """ + return cls() + + @classmethod + def load(cls, customize_path): + obj = cls() + + logger.debug("Searching %r for files to load", customize_path) + + customize_files = glob.glob(f"{customize_path}/**/*.py", recursive=True) + + # filter out any tests + customize_files = [ + x for x in customize_files if "tests" not in x and "conftest" not in x + ] + + customize_module_spec = importlib.machinery.ModuleSpec("cime_customize", None) + + customize_module = importlib.util.module_from_spec(customize_module_spec) + + sys.modules["CIME.customize"] = customize_module + + for x in sorted(customize_files): + obj._load_file(x, customize_module) + + setattr(obj, "_loaded", True) + + return obj + + def _load_file(self, file_path, customize_module): + logger.debug("Loading file %r", file_path) + + raw_config = utils.import_from_file("raw_config", file_path) + + # filter user define variables and functions + user_defined = [x for x in dir(raw_config) if not x.endswith("__")] + + # set values on this object, will overwrite existing + for x in user_defined: + try: + value = getattr(raw_config, x) + except AttributeError: + # should never hit this + logger.fatal("Attribute %r missing on obejct", x) + + sys.exit(1) + else: + setattr(customize_module, x, value) + + self._set_attribute(x, value) + + def _set_attribute(self, name, value, desc=None): + if hasattr(self, name): + logger.debug("Overwriting %r attribute", name) + + logger.debug("Setting attribute %r with value %r", name, value) + + setattr(self, name, value) + + self._attribute_config[name] = { + "desc": desc, + "default": value, + } + + def print_rst_table(self): + max_variable = max([len(x) for x in self._attribute_config.keys()]) + max_default = max( + [len(str(x["default"])) for x in self._attribute_config.values()] + ) + max_type = max( + [len(type(x["default"]).__name__) for x in self._attribute_config.values()] + ) + max_desc = max([len(x["desc"]) for x in self._attribute_config.values()]) + + divider_row = ( + f"{'='*max_variable} {'='*max_default} {'='*max_type} {'='*max_desc}" + ) + + rows = [ + divider_row, + f"Variable{' '*(max_variable-8)} Default{' '*(max_default-7)} Type{' '*(max_type-4)} Description{' '*(max_desc-11)}", + divider_row, + ] + + for variable, value in sorted( + self._attribute_config.items(), key=lambda x: x[0] + ): + variable_fill = max_variable - len(variable) + default_fill = max_default - len(str(value["default"])) + type_fill = max_type - len(type(value["default"]).__name__) + + rows.append( + f"{variable}{' '*variable_fill} {value['default']}{' '*default_fill} {type(value['default']).__name__}{' '*type_fill} {value['desc']}" + ) + + rows.append(divider_row) + + print("\n".join(rows)) diff --git a/CIME/hist_utils.py b/CIME/hist_utils.py index e001f5b8102..ed2cbebc81a 100644 --- a/CIME/hist_utils.py +++ b/CIME/hist_utils.py @@ -2,11 +2,11 @@ Functions for actions pertaining to history files. """ from CIME.XML.standard_module_setup import * +from CIME.config import Config from CIME.test_status import TEST_NO_BASELINES_COMMENT, TEST_STATUS_FILENAME from CIME.utils import ( get_current_commit, get_timestamp, - get_model, safe_copy, SharedArea, parse_test_name, @@ -520,7 +520,7 @@ def compare_baseline(case, baseline_dir=None, outfile_suffix=""): success, comments = _compare_hists( case, rundir, basecmp_dir, outfile_suffix=outfile_suffix ) - if get_model() == "e3sm": + if Config.instance().create_bless_log: bless_log = os.path.join(basecmp_dir, BLESS_LOG_NAME) if os.path.exists(bless_log): lines = open(bless_log, "r", encoding="utf-8").readlines() @@ -536,23 +536,22 @@ def generate_teststatus(testdir, baseline_dir): CESM stores it's TestStatus file in baselines. Do not let exceptions escape from this function. """ - if get_model() == "cesm": - try: - with SharedArea(): - if not os.path.isdir(baseline_dir): - os.makedirs(baseline_dir) - - safe_copy( - os.path.join(testdir, TEST_STATUS_FILENAME), - baseline_dir, - preserve_meta=False, - ) - except Exception as e: - logger.warning( - "Could not copy {} to baselines, {}".format( - os.path.join(testdir, TEST_STATUS_FILENAME), str(e) - ) + try: + with SharedArea(): + if not os.path.isdir(baseline_dir): + os.makedirs(baseline_dir) + + safe_copy( + os.path.join(testdir, TEST_STATUS_FILENAME), + baseline_dir, + preserve_meta=False, ) + except Exception as e: + logger.warning( + "Could not copy {} to baselines, {}".format( + os.path.join(testdir, TEST_STATUS_FILENAME), str(e) + ) + ) def _generate_baseline_impl(case, baseline_dir=None, allow_baseline_overwrite=False): @@ -646,7 +645,7 @@ def _generate_baseline_impl(case, baseline_dir=None, allow_baseline_overwrite=Fa ), ) - if get_model() == "e3sm": + if Config.instance().create_bless_log: bless_log = os.path.join(basegen_dir, BLESS_LOG_NAME) with open(bless_log, "a", encoding="utf-8") as fd: fd.write( diff --git a/CIME/provenance.py b/CIME/provenance.py index a7dfa24a688..849bb2757d1 100644 --- a/CIME/provenance.py +++ b/CIME/provenance.py @@ -6,784 +6,17 @@ from CIME.XML.standard_module_setup import * from CIME.utils import ( - touch, - gzip_existing_file, SharedArea, convert_to_babylonian_time, get_current_commit, - get_current_submodule_status, - indent_string, run_cmd, - run_cmd_no_fail, - safe_copy, - copy_globs, ) -import tarfile, getpass, signal, glob, shutil, sys -from contextlib import contextmanager +import sys logger = logging.getLogger(__name__) -def _get_batch_job_id_for_syslog(case): - """ - mach_syslog only works on certain machines - """ - mach = case.get_value("MACH") - try: - if mach in ["anvil", "chrysalis", "compy", "cori-haswell", "cori-knl"]: - return os.environ["SLURM_JOB_ID"] - elif mach in ["theta"]: - return os.environ["COBALT_JOBID"] - elif mach in ["summit"]: - return os.environ["LSB_JOBID"] - except KeyError: - pass - - return None - - -def _extract_times(zipfiles, target_file): - - contents = "Target Build_time\n" - total_build_time = 0.0 - for zipfile in zipfiles: - stat, output, _ = run_cmd("zgrep 'built in' {}".format(zipfile)) - if stat == 0: - for line in output.splitlines(): - line = line.strip() - if line: - items = line.split() - target, the_time = items[1], items[-2] - contents += "{} {}\n".format(target, the_time) - - stat, output, _ = run_cmd("zgrep -E '^real [0-9.]+$' {}".format(zipfile)) - if stat == 0: - for line in output.splitlines(): - line = line.strip() - if line: - total_build_time += float(line.split()[-1]) - - with open(target_file, "w") as fd: - fd.write(contents) - fd.write("Total_Elapsed_Time {}".format(str(total_build_time))) - - -def _run_git_cmd_recursively(cmd, srcroot, output): - """Runs a git command recursively - - Runs the git command in srcroot then runs it on each submodule. - Then output from both commands is written to the output file. - """ - rc1, output1, err1 = run_cmd("git {}".format(cmd), from_dir=srcroot) - - rc2, output2, err2 = run_cmd( - 'git submodule foreach --recursive "git {}; echo"'.format(cmd), from_dir=srcroot - ) - - with open(output, "w") as fd: - fd.write((output1 if rc1 == 0 else err1) + "\n\n") - fd.write((output2 if rc2 == 0 else err2) + "\n") - - -def _parse_dot_git_path(gitdir): - """Parse `.git` path. - - Take a path e.g. `/storage/cime/.git/worktrees/cime` and parse `.git` - directory e.g. `/storage/cime/.git`. - - Args: - gitdir (str): Path containing `.git` directory. - - Returns: - str: Path ending with `.git` - """ - dot_git_pattern = r"^(.*/\.git).*" - - m = re.match(dot_git_pattern, gitdir) - - expect(m is not None, f"Could not parse .git from path {gitdir!r}") - - return m.group(1) - - -def _read_gitdir(gitroot): - """Read `gitdir` from `.git` file. - - Reads `.git` file in a worktree or submodule and parse `gitdir`. - - Args: - gitroot (str): Path ending with `.git` file. - - Returns: - str: Path contained in `.git` file. - """ - expect(os.path.isfile(gitroot), f"Expected {gitroot!r} to be a file") - - with open(gitroot) as fd: - line = fd.readline() - - gitdir_pattern = r"^gitdir:\s*(.*)$" - - m = re.match(gitdir_pattern, line) - - expect(m is not None, f"Could parse gitdir path from file {gitroot!r}") - - return m.group(1) - - -@contextmanager -def _swap_cwd(new_cwd): - old_cwd = os.getcwd() - - os.chdir(new_cwd) - - try: - yield - finally: - os.chdir(old_cwd) - - -def _find_git_root(srcroot): - """Finds the `.git` directory. - - NOTICE: It's assumed `srcroot` is an absolute path. - - There are three scenarios to locate the correct `.git` directory: - - NOTICE: In the 3rd case git `.git` directory is not actually `.git`. - - 1. In a standard git repository it will be located at `{srcroot}/.git`. - 2. In a git worktree the `{srcroot}/.git` is a file containing a path, - `{gitdir}`, which the `{gitroot}` can be parsed from. - 3. In a git submodule the `{srcroot}/.git` is a file containing a path, - `{gitdir}`, where the `{gitroot}` is `{gitdir}`. - - To aid in finding the correct `{gitroot}` the file `{gitroot}/config` - is checked, this file will always be located in the correct directory. - - Args: - srcroot (str): Path to the source root. - - Returns: - str: Absolute path to `.git` directory. - """ - gitroot = f"{srcroot}/.git" - - expect( - os.path.exists(gitroot), - f"{srcroot!r} is not a git repository, failed to collect provenance", - ) - - # Handle 1st scenario - if os.path.isdir(gitroot): - return gitroot - - # ensure we're in correct directory so abspath works correctly - with _swap_cwd(srcroot): - gitdir = os.path.abspath(_read_gitdir(gitroot)) - - # Handle 3rd scenario, gitdir is the `.git` directory - config_path = os.path.join(gitdir, "config") - - if os.path.exists(config_path): - return gitdir - - # Handle 2nd scenario, gitdir should already be absolute path - parsed_gitroot = _parse_dot_git_path(gitdir) - - return parsed_gitroot - - -def _record_git_provenance(srcroot, exeroot, lid): - """Records git provenance - - Records git status, diff and logs for main repo and all submodules. - """ - # Git Status - status_prov = os.path.join(exeroot, "GIT_STATUS.{}".format(lid)) - _run_git_cmd_recursively("status", srcroot, status_prov) - - # Git Diff - diff_prov = os.path.join(exeroot, "GIT_DIFF.{}".format(lid)) - _run_git_cmd_recursively("diff", srcroot, diff_prov) - - # Git Log - log_prov = os.path.join(exeroot, "GIT_LOG.{}".format(lid)) - cmd = "log --first-parent --pretty=oneline -n 5" - _run_git_cmd_recursively(cmd, srcroot, log_prov) - - # Git remote - remote_prov = os.path.join(exeroot, "GIT_REMOTE.{}".format(lid)) - _run_git_cmd_recursively("remote -v", srcroot, remote_prov) - - gitroot = _find_git_root(srcroot) - - # Git config - config_src = os.path.join(gitroot, "config") - config_prov = os.path.join(exeroot, "GIT_CONFIG.{}".format(lid)) - safe_copy(config_src, config_prov, preserve_meta=False) - - -def _save_build_provenance_e3sm(case, lid): - srcroot = case.get_value("SRCROOT") - exeroot = case.get_value("EXEROOT") - caseroot = case.get_value("CASEROOT") - - # Save git describe - describe_prov = os.path.join(exeroot, "GIT_DESCRIBE.{}".format(lid)) - desc = get_current_commit(tag=True, repo=srcroot) - with open(describe_prov, "w") as fd: - fd.write(desc) - - gitroot = _find_git_root(srcroot) - - # Save HEAD - headfile = os.path.join(gitroot, "logs", "HEAD") - headfile_prov = os.path.join(exeroot, "GIT_LOGS_HEAD.{}".format(lid)) - if os.path.exists(headfile_prov): - os.remove(headfile_prov) - if os.path.exists(headfile): - safe_copy(headfile, headfile_prov, preserve_meta=False) - - # Save git submodule status - submodule_prov = os.path.join(exeroot, "GIT_SUBMODULE_STATUS.{}".format(lid)) - subm_status = get_current_submodule_status(recursive=True, repo=srcroot) - with open(submodule_prov, "w") as fd: - fd.write(subm_status) - - _record_git_provenance(srcroot, exeroot, lid) - - # Save SourceMods - sourcemods = os.path.join(caseroot, "SourceMods") - sourcemods_prov = os.path.join(exeroot, "SourceMods.{}.tar.gz".format(lid)) - if os.path.exists(sourcemods_prov): - os.remove(sourcemods_prov) - if os.path.isdir(sourcemods): - with tarfile.open(sourcemods_prov, "w:gz") as tfd: - tfd.add(sourcemods, arcname="SourceMods") - - # Save build env - env_prov = os.path.join(exeroot, "build_environment.{}.txt".format(lid)) - if os.path.exists(env_prov): - os.remove(env_prov) - env_module = case.get_env("mach_specific") - env_module.save_all_env_info(env_prov) - - # Save build times - build_times = os.path.join(exeroot, "build_times.{}.txt".format(lid)) - if os.path.exists(build_times): - os.remove(build_times) - globstr = "{}/*bldlog*{}.gz".format(exeroot, lid) - matches = glob.glob(globstr) - if matches: - _extract_times(matches, build_times) - - # For all the just-created post-build provenance files, symlink a generic name - # to them to indicate that these are the most recent or active. - for item in [ - "GIT_DESCRIBE", - "GIT_LOGS_HEAD", - "GIT_SUBMODULE_STATUS", - "GIT_STATUS", - "GIT_DIFF", - "GIT_LOG", - "GIT_CONFIG", - "GIT_REMOTE", - "SourceMods", - "build_environment", - "build_times", - ]: - globstr = "{}/{}.{}*".format(exeroot, item, lid) - matches = glob.glob(globstr) - expect( - len(matches) < 2, - "Multiple matches for glob {} should not have happened".format(globstr), - ) - if matches: - the_match = matches[0] - generic_name = the_match.replace(".{}".format(lid), "") - if os.path.exists(generic_name): - os.remove(generic_name) - os.symlink(the_match, generic_name) - - -def _save_build_provenance_cesm(case, lid): # pylint: disable=unused-argument - version = case.get_value("MODEL_VERSION") - # version has already been recorded - srcroot = case.get_value("SRCROOT") - manic = os.path.join("manage_externals", "checkout_externals") - manic_full_path = os.path.join(srcroot, manic) - out = None - if os.path.exists(manic_full_path): - args = " --status --verbose --no-logging" - stat, out, err = run_cmd(manic_full_path + args, from_dir=srcroot) - errmsg = """Error gathering provenance information from manage_externals. - -manage_externals error message: -{err} - -manage_externals output: -{out} - -To solve this, either: - -(1) Find and fix the problem: From {srcroot}, try to get this command to work: - {manic}{args} - -(2) If you don't need provenance information, rebuild with --skip-provenance-check -""".format( - out=indent_string(out, 4), - err=indent_string(err, 4), - srcroot=srcroot, - manic=manic, - args=args, - ) - expect(stat == 0, errmsg) - - caseroot = case.get_value("CASEROOT") - with open(os.path.join(caseroot, "CaseStatus"), "a") as fd: - if version is not None and version != "unknown": - fd.write("CESM version is {}\n".format(version)) - if out is not None: - fd.write("{}\n".format(out)) - - -def save_build_provenance(case, lid=None): - with SharedArea(): - model = case.get_value("MODEL") - lid = os.environ["LID"] if lid is None else lid - - if model == "e3sm": - _save_build_provenance_e3sm(case, lid) - elif model == "cesm": - _save_build_provenance_cesm(case, lid) - - -def _save_prerun_timing_e3sm(case, lid): - project = case.get_value("PROJECT", subgroup=case.get_primary_job()) - if not case.is_save_timing_dir_project(project): - return - - timing_dir = case.get_value("SAVE_TIMING_DIR") - if timing_dir is None or not os.path.isdir(timing_dir): - logger.warning( - "SAVE_TIMING_DIR {} is not valid. E3SM requires a valid SAVE_TIMING_DIR to archive timing data.".format( - timing_dir - ) - ) - return - - logger.info( - "Archiving timing data and associated provenance in {}.".format(timing_dir) - ) - rundir = case.get_value("RUNDIR") - blddir = case.get_value("EXEROOT") - caseroot = case.get_value("CASEROOT") - srcroot = case.get_value("SRCROOT") - base_case = case.get_value("CASE") - full_timing_dir = os.path.join( - timing_dir, "performance_archive", getpass.getuser(), base_case, lid - ) - if os.path.exists(full_timing_dir): - logger.warning( - "{} already exists. Skipping archive of timing data and associated provenance.".format( - full_timing_dir - ) - ) - return - - try: - os.makedirs(full_timing_dir) - except OSError: - logger.warning( - "{} cannot be created. Skipping archive of timing data and associated provenance.".format( - full_timing_dir - ) - ) - return - - mach = case.get_value("MACH") - compiler = case.get_value("COMPILER") - - # For some batch machines save queue info - job_id = _get_batch_job_id_for_syslog(case) - if job_id is not None: - if mach == "theta": - for cmd, filename in [ - ( - "qstat -l --header JobID:JobName:User:Project:WallTime:QueuedTime:Score:RunTime:TimeRemaining:Nodes:State:Location:Mode:Command:Args:Procs:Queue:StartTime:attrs:Geometry", - "qstatf", - ), - ("qstat -lf %s" % job_id, "qstatf_jobid"), - ("xtnodestat", "xtnodestat"), - ("xtprocadmin", "xtprocadmin"), - ]: - filename = "%s.%s" % (filename, lid) - run_cmd_no_fail(cmd, arg_stdout=filename, from_dir=full_timing_dir) - gzip_existing_file(os.path.join(full_timing_dir, filename)) - elif mach in ["cori-haswell", "cori-knl"]: - for cmd, filename in [ - ("sinfo -a -l", "sinfol"), - ("scontrol show jobid %s" % job_id, "sqsf_jobid"), - # ("sqs -f", "sqsf"), - ( - "squeue -o '%.10i %.15P %.20j %.10u %.7a %.2t %.6D %.8C %.10M %.10l %.20S %.20V'", - "squeuef", - ), - ("squeue -t R -o '%.10i %R'", "squeues"), - ]: - filename = "%s.%s" % (filename, lid) - run_cmd_no_fail(cmd, arg_stdout=filename, from_dir=full_timing_dir) - gzip_existing_file(os.path.join(full_timing_dir, filename)) - elif mach in ["anvil", "chrysalis", "compy"]: - for cmd, filename in [ - ("sinfo -l", "sinfol"), - ("squeue -o '%all' --job {}".format(job_id), "squeueall_jobid"), - ( - "squeue -o '%.10i %.10P %.15u %.20a %.2t %.6D %.8C %.12M %.12l %.20S %.20V %j'", - "squeuef", - ), - ("squeue -t R -o '%.10i %R'", "squeues"), - ]: - filename = "%s.%s" % (filename, lid) - run_cmd_no_fail(cmd, arg_stdout=filename, from_dir=full_timing_dir) - gzip_existing_file(os.path.join(full_timing_dir, filename)) - elif mach == "summit": - for cmd, filename in [ - ("bjobs -u all >", "bjobsu_all"), - ("bjobs -r -u all -o 'jobid slots exec_host' >", "bjobsru_allo"), - ("bjobs -l -UF %s >" % job_id, "bjobslUF_jobid"), - ]: - full_cmd = cmd + " " + filename - run_cmd_no_fail(full_cmd + "." + lid, from_dir=full_timing_dir) - gzip_existing_file(os.path.join(full_timing_dir, filename + "." + lid)) - - # copy/tar SourceModes - source_mods_dir = os.path.join(caseroot, "SourceMods") - if os.path.isdir(source_mods_dir): - with tarfile.open( - os.path.join(full_timing_dir, "SourceMods.{}.tar.gz".format(lid)), "w:gz" - ) as tfd: - tfd.add(source_mods_dir, arcname="SourceMods") - - # Save various case configuration items - case_docs = os.path.join(full_timing_dir, "CaseDocs.{}".format(lid)) - os.mkdir(case_docs) - globs_to_copy = [ - "CaseDocs/*", - "run_script_provenance/*", - "*.run", - ".*.run", - "*.xml", - "user_nl_*", - "*env_mach_specific*", - "Macros*", - "README.case", - "Depends.{}".format(mach), - "Depends.{}".format(compiler), - "Depends.{}.{}".format(mach, compiler), - "software_environment.txt", - ] - - copy_globs([os.path.join(caseroot, x) for x in globs_to_copy], case_docs, lid) - - # Copy some items from build provenance - blddir_globs_to_copy = [ - "GIT_LOGS_HEAD", - "GIT_STATUS", - "GIT_DIFF", - "GIT_LOG", - "GIT_REMOTE", - "GIT_CONFIG", - "GIT_SUBMODULE_STATUS", - "build_environment.txt", - "build_times.txt", - ] - - copy_globs( - [os.path.join(blddir, x) for x in blddir_globs_to_copy], full_timing_dir, lid - ) - - rundir_globs_to_copy = [ - "preview_run.log", - ] - - copy_globs( - [os.path.join(rundir, x) for x in rundir_globs_to_copy], full_timing_dir, lid - ) - - # Save state of repo - from_repo = ( - srcroot - if os.path.exists(os.path.join(srcroot, ".git")) - else os.path.dirname(srcroot) - ) - desc = get_current_commit(tag=True, repo=from_repo) - with open(os.path.join(full_timing_dir, "GIT_DESCRIBE.{}".format(lid)), "w") as fd: - fd.write(desc) - - # What this block does is mysterious to me (JGF) - if job_id is not None: - - # Kill mach_syslog from previous run if one exists - syslog_jobid_path = os.path.join(rundir, "syslog_jobid.{}".format(job_id)) - if os.path.exists(syslog_jobid_path): - try: - with open(syslog_jobid_path, "r") as fd: - syslog_jobid = int(fd.read().strip()) - os.kill(syslog_jobid, signal.SIGTERM) - except (ValueError, OSError) as e: - logger.warning("Failed to kill syslog: {}".format(e)) - finally: - os.remove(syslog_jobid_path) - - # If requested, spawn a mach_syslog process to monitor job progress - sample_interval = case.get_value("SYSLOG_N") - if sample_interval > 0: - archive_checkpoints = os.path.join( - full_timing_dir, "checkpoints.{}".format(lid) - ) - os.mkdir(archive_checkpoints) - touch("{}/e3sm.log.{}".format(rundir, lid)) - syslog_jobid = run_cmd_no_fail( - "./mach_syslog {si} {jobid} {lid} {rundir} {rundir}/timing/checkpoints {ac} >& /dev/null & echo $!".format( - si=sample_interval, - jobid=job_id, - lid=lid, - rundir=rundir, - ac=archive_checkpoints, - ), - from_dir=os.path.join(caseroot, "Tools"), - ) - with open( - os.path.join(rundir, "syslog_jobid.{}".format(job_id)), "w" - ) as fd: - fd.write("{}\n".format(syslog_jobid)) - - -def _cleanup_spio_stats(case): - rundir = case.get_value("RUNDIR") - for item in glob.glob(os.path.join(rundir, "io_perf_summary*")): - os.remove(item) - - spio_stats_dir = os.path.join(rundir, "spio_stats") - if os.path.exists(spio_stats_dir): - shutil.rmtree(spio_stats_dir) - - try: - os.makedirs(spio_stats_dir) - except OSError: - logger.warning( - "{} could not be created. Scorpio I/O statistics will be stored in the run directory.".format( - spio_stats_dir - ) - ) - - -def _save_prerun_provenance_e3sm(case, lid): - _cleanup_spio_stats(case) - if case.get_value("SAVE_TIMING"): - _save_prerun_timing_e3sm(case, lid) - - -def _save_prerun_provenance_cesm(case, lid): # pylint: disable=unused-argument - pass - - -def _save_prerun_provenance_common(case, lid): - """Saves common prerun provenance.""" - run_dir = case.get_value("RUNDIR") - - base_preview_run = os.path.join(run_dir, "preview_run.log") - preview_run = f"{base_preview_run}.{lid}" - - if os.path.exists(base_preview_run): - os.remove(base_preview_run) - - with open(base_preview_run, "w") as fd: - case.preview_run(lambda x: fd.write("{}\n".format(x)), None) - - # Create copy rather than symlink, the log is automatically gzipped - safe_copy(base_preview_run, preview_run) - - -def save_prerun_provenance(case, lid=None): - with SharedArea(): - # Always save env - lid = os.environ["LID"] if lid is None else lid - env_module = case.get_env("mach_specific") - logdir = os.path.join(case.get_value("CASEROOT"), "logs") - if not os.path.isdir(logdir): - os.makedirs(logdir) - env_module.save_all_env_info( - os.path.join(logdir, "run_environment.txt.{}".format(lid)) - ) - - _save_prerun_provenance_common(case, lid) - - model = case.get_value("MODEL") - if model == "e3sm": - _save_prerun_provenance_e3sm(case, lid) - elif model == "cesm": - _save_prerun_provenance_cesm(case, lid) - - -def _save_postrun_provenance_cesm(case, lid): - save_timing = case.get_value("SAVE_TIMING") - if save_timing: - rundir = case.get_value("RUNDIR") - timing_dir = os.path.join("timing", case.get_value("CASE")) - shutil.move( - os.path.join(rundir, "timing"), os.path.join(timing_dir, "timing." + lid) - ) - - -def _save_postrun_timing_e3sm(case, lid): - caseroot = case.get_value("CASEROOT") - rundir = case.get_value("RUNDIR") - - # tar timings - rundir_timing_dir = os.path.join(rundir, "timing." + lid) - shutil.move(os.path.join(rundir, "timing"), rundir_timing_dir) - with tarfile.open("%s.tar.gz" % rundir_timing_dir, "w:gz") as tfd: - tfd.add(rundir_timing_dir, arcname=os.path.basename(rundir_timing_dir)) - - shutil.rmtree(rundir_timing_dir) - - atm_chunk_costs_src_path = os.path.join(rundir, "atm_chunk_costs.txt") - if os.path.exists(atm_chunk_costs_src_path): - atm_chunk_costs_dst_path = os.path.join( - rundir, "atm_chunk_costs.{}".format(lid) - ) - shutil.move(atm_chunk_costs_src_path, atm_chunk_costs_dst_path) - gzip_existing_file(atm_chunk_costs_dst_path) - - # gzip memory profile log - glob_to_copy = "memory.[0-4].*.log" - for item in glob.glob(os.path.join(rundir, glob_to_copy)): - mprof_dst_path = os.path.join( - os.path.dirname(item), (os.path.basename(item) + ".{}").format(lid) - ) - shutil.move(item, mprof_dst_path) - gzip_existing_file(mprof_dst_path) - - # Copy Scorpio I/O performance stats in "spio_stats" to "spio_stats.[LID]" + tar + compress - spio_stats_dir = os.path.join(rundir, "spio_stats") - if not os.path.exists(spio_stats_dir): - os.mkdir(spio_stats_dir) - - for item in glob.glob(os.path.join(rundir, "io_perf_summary*")): - safe_copy(item, spio_stats_dir) - - spio_stats_job_dir = os.path.join(rundir, "spio_stats." + lid) - shutil.copytree(spio_stats_dir, spio_stats_job_dir) - with tarfile.open("%s.tar.gz" % spio_stats_job_dir, "w:gz") as tfd: - tfd.add(spio_stats_job_dir, arcname=os.path.basename(spio_stats_job_dir)) - - shutil.rmtree(spio_stats_job_dir) - - gzip_existing_file(os.path.join(caseroot, "timing", "e3sm_timing_stats.%s" % lid)) - - # JGF: not sure why we do this - timing_saved_file = "timing.%s.saved" % lid - touch(os.path.join(caseroot, "timing", timing_saved_file)) - - project = case.get_value("PROJECT", subgroup=case.get_primary_job()) - if not case.is_save_timing_dir_project(project): - return - - timing_dir = case.get_value("SAVE_TIMING_DIR") - if timing_dir is None or not os.path.isdir(timing_dir): - return - - mach = case.get_value("MACH") - base_case = case.get_value("CASE") - full_timing_dir = os.path.join( - timing_dir, "performance_archive", getpass.getuser(), base_case, lid - ) - - if not os.path.isdir(full_timing_dir): - return - - # Kill mach_syslog - job_id = _get_batch_job_id_for_syslog(case) - if job_id is not None: - syslog_jobid_path = os.path.join(rundir, "syslog_jobid.{}".format(job_id)) - if os.path.exists(syslog_jobid_path): - try: - with open(syslog_jobid_path, "r") as fd: - syslog_jobid = int(fd.read().strip()) - os.kill(syslog_jobid, signal.SIGTERM) - except (ValueError, OSError) as e: - logger.warning("Failed to kill syslog: {}".format(e)) - finally: - os.remove(syslog_jobid_path) - - # copy timings - safe_copy("%s.tar.gz" % rundir_timing_dir, full_timing_dir, preserve_meta=False) - - # - # save output files and logs - # - globs_to_copy = [] - if job_id is not None: - if mach in ["anvil", "chrysalis", "compy", "cori-haswell", "cori-knl"]: - globs_to_copy.append("run*%s*%s" % (case.get_value("CASE"), job_id)) - elif mach == "theta": - globs_to_copy.append("%s*error" % job_id) - globs_to_copy.append("%s*output" % job_id) - globs_to_copy.append("%s*cobaltlog" % job_id) - elif mach == "summit": - globs_to_copy.append("e3sm.stderr.%s" % job_id) - globs_to_copy.append("e3sm.stdout.%s" % job_id) - - globs_to_copy.append("logs/run_environment.txt.{}".format(lid)) - globs_to_copy.append(os.path.join(rundir, "e3sm.log.{}.gz".format(lid))) - globs_to_copy.append(os.path.join(rundir, "cpl.log.{}.gz".format(lid))) - globs_to_copy.append(os.path.join(rundir, "atm_chunk_costs.{}.gz".format(lid))) - globs_to_copy.append(os.path.join(rundir, "memory.[0-4].*.log.{}.gz".format(lid))) - globs_to_copy.append("timing/*.{}*".format(lid)) - globs_to_copy.append("CaseStatus") - globs_to_copy.append(os.path.join(rundir, "spio_stats.{}.tar.gz".format(lid))) - globs_to_copy.append(os.path.join(caseroot, "replay.sh")) - - for glob_to_copy in globs_to_copy: - for item in glob.glob(os.path.join(caseroot, glob_to_copy)): - basename = os.path.basename(item) - if basename != timing_saved_file: - if lid not in basename and not basename.endswith(".gz"): - safe_copy( - item, - os.path.join(full_timing_dir, "{}.{}".format(basename, lid)), - preserve_meta=False, - ) - else: - safe_copy(item, full_timing_dir, preserve_meta=False) - - # zip everything - for root, _, files in os.walk(full_timing_dir): - for filename in files: - if not filename.endswith(".gz"): - gzip_existing_file(os.path.join(root, filename)) - - -def _save_postrun_provenance_e3sm(case, lid): - if case.get_value("SAVE_TIMING"): - _save_postrun_timing_e3sm(case, lid) - - -def save_postrun_provenance(case, lid=None): - with SharedArea(): - model = case.get_value("MODEL") - lid = os.environ["LID"] if lid is None else lid - - if model == "e3sm": - _save_postrun_provenance_e3sm(case, lid) - elif model == "cesm": - _save_postrun_provenance_cesm(case, lid) - - _WALLTIME_BASELINE_NAME = "walltimes" _WALLTIME_FILE_NAME = "walltimes" _GLOBAL_MINUMUM_TIME = 900 diff --git a/CIME/scripts/create_newcase.py b/CIME/scripts/create_newcase.py index 744ee603233..5d4d2743da3 100755 --- a/CIME/scripts/create_newcase.py +++ b/CIME/scripts/create_newcase.py @@ -9,11 +9,11 @@ from CIME.Tools.standard_script_setup import * from CIME.utils import ( expect, - get_model, get_cime_config, get_cime_default_driver, get_src_root, ) +from CIME.config import Config from CIME.case import Case from argparse import RawTextHelpFormatter @@ -28,6 +28,10 @@ def parse_command_line(args, cimeroot, description): CIME.utils.setup_standard_logging_options(parser) + customize_path = os.path.join(CIME.utils.get_src_root(), "cime_config", "customize") + + config = Config.load(customize_path) + try: cime_config = get_cime_config() except Exception: @@ -166,11 +170,6 @@ def parse_command_line(args, cimeroot, description): help="A workflow from config_workflow.xml to apply to this case. ", ) - if cime_config: - model = get_model() - else: - model = None - srcroot_default = get_src_root() parser.add_argument( @@ -190,7 +189,7 @@ def parse_command_line(args, cimeroot, description): "--script-root", dest="script_root", default=None, help=argparse.SUPPRESS ) - if model == "cesm": + if config.allow_unsupported: parser.add_argument( "--run-unsupported", action="store_true", @@ -231,25 +230,19 @@ def parse_command_line(args, cimeroot, description): "--input-dir", help="Use a non-default location for input files. This will change the xml value of DIN_LOC_ROOT.", ) - if model == "cesm": - drv_choices = ("mct", "nuopc") - drv_help = ( - "Override the top level driver type and use this one " - + "(changes xml variable COMP_INTERFACE) [this is an advanced option]" - ) - elif model == "e3sm": - drv_choices = ("mct", "moab") - drv_help = argparse.SUPPRESS - else: - drv_choices = None - if drv_choices is not None: - parser.add_argument( - "--driver", - default=get_cime_default_driver(), - choices=drv_choices, - help=drv_help, - ) + drv_choices = config.driver_choices + drv_help = ( + "Override the top level driver type and use this one " + + "(changes xml variable COMP_INTERFACE) [this is an advanced option]" + ) + + parser.add_argument( + "--driver", + default=get_cime_default_driver(), + choices=drv_choices, + help=drv_help, + ) parser.add_argument( "-n", @@ -300,7 +293,7 @@ def parse_command_line(args, cimeroot, description): ) run_unsupported = False - if model == "cesm": + if config.allow_unsupported: run_unsupported = args.run_unsupported expect( @@ -313,7 +306,7 @@ def parse_command_line(args, cimeroot, description): elif cime_config and cime_config.has_option("main", "input_dir"): args.input_dir = os.path.abspath(cime_config.get("main", "input_dir")) - if model == "cesm" and args.driver == "mct": + if config.create_test_flag_mode == "cesm" and args.driver == "mct": logger.warning( """======================================================================== WARNING: The MCT-based driver and data models will be removed from CESM diff --git a/CIME/scripts/create_test.py b/CIME/scripts/create_test.py index 6f89ee0324e..53ee90f170d 100755 --- a/CIME/scripts/create_test.py +++ b/CIME/scripts/create_test.py @@ -38,6 +38,7 @@ run_cmd_no_fail, get_cime_config, ) +from CIME.config import Config from CIME.XML.machines import Machines from CIME.case import Case from CIME.test_utils import get_tests_from_xml @@ -55,7 +56,7 @@ def parse_command_line(args, description): description=description, formatter_class=RawTextHelpFormatter ) - model = CIME.utils.get_model() + model_config = Config.instance() CIME.utils.setup_standard_logging_options(parser) @@ -179,7 +180,7 @@ def parse_command_line(args, description): "\ninvoke ./query_config. The default is the first listing .", ) - if model in ["cesm", "ufs"]: + if model_config.create_test_flag_mode == "cesm": parser.add_argument( "-c", "--compare", @@ -479,7 +480,7 @@ def parse_command_line(args, description): CIME.utils.resolve_mail_type_args(args) # generate and compare flags may not point to the same directory - if model in ["cesm", "ufs"]: + if model_config.create_test_flag_mode == "cesm": if args.generate is not None: expect( not (args.generate == args.compare), @@ -577,7 +578,7 @@ def parse_command_line(args, description): # Compute list of fully-resolved test_names test_extra_data = {} - if model in ["cesm", "ufs"]: + if model_config.check_machine_name_from_test_name: machine_name = args.xml_machine if args.machine is None else args.machine # If it's still unclear what machine to use, look at test names @@ -661,7 +662,7 @@ def parse_command_line(args, description): baseline_cmp_name = None baseline_gen_name = None if args.compare or args.generate: - if model in ["cesm", "ufs"]: + if model_config.create_test_flag_mode == "cesm": if args.compare is not None: baseline_cmp_name = args.compare if args.generate is not None: @@ -690,7 +691,7 @@ def parse_command_line(args, description): expect(dot_count > 1 and dot_count <= 4, "Invalid test Name, '{}'".format(name)) # for e3sm, sort by walltime - if model == "e3sm": + if model_config.sort_tests: if args.walltime is None: # Longest tests should run first test_names.sort(key=get_tests.key_test_time, reverse=True) @@ -992,6 +993,11 @@ def create_test( ############################################################################### def _main_func(description=None): ############################################################################### + customize_path = os.path.join(utils.get_src_root(), "cime_config", "customize") + + if os.path.exists(customize_path): + Config.instance().load(customize_path) + ( test_names, test_data, diff --git a/CIME/scripts/query_config.py b/CIME/scripts/query_config.py index 0cb8f01fd9e..674713e0485 100755 --- a/CIME/scripts/query_config.py +++ b/CIME/scripts/query_config.py @@ -8,18 +8,24 @@ from CIME.Tools.standard_script_setup import * import re -from CIME.utils import expect, get_model +from CIME.utils import expect from CIME.XML.files import Files from CIME.XML.component import Component from CIME.XML.compsets import Compsets from CIME.XML.grids import Grids +from CIME.config import Config # from CIME.XML.machines import Machines import CIME.XML.machines from argparse import RawTextHelpFormatter logger = logging.getLogger(__name__) -supported_comp_interfaces = ["mct", "nuopc", "moab"] + +customize_path = os.path.join(CIME.utils.get_src_root(), "cime_config", "customize") + +config = Config.load(customize_path) + +supported_comp_interfaces = list(config.driver_choices) def query_grids(files, long_output, xml=False): @@ -122,7 +128,7 @@ def print_compset(name, files, all_components=False, xml=False): elif config_file is None or not os.path.isfile(config_file): return - if get_model() == "ufs" and name == "drv": + if config.test_mode not in ("e3sm", "cesm") and name == "drv": return print("\nActive component: {}".format(name)) @@ -256,10 +262,7 @@ def parse_command_line(args, description): supported_comp_interfaces.remove(comp_interface) for comp in components: - if cime_model == "cesm": - string = "COMP_ROOT_DIR_{}".format(comp) - else: - string = "CONFIG_{}_FILE".format(comp) + string = config.xml_component_key.format(comp) # determine all components in string components = files[comp_interface].get_components(string) diff --git a/CIME/test_scheduler.py b/CIME/test_scheduler.py index 58a68d77483..d46d9e0d3b1 100644 --- a/CIME/test_scheduler.py +++ b/CIME/test_scheduler.py @@ -31,6 +31,7 @@ get_cime_default_driver, clear_folder, ) +from CIME.config import Config from CIME.test_status import * from CIME.XML.machines import Machines from CIME.XML.generic_xml import GenericXML @@ -233,7 +234,9 @@ def __init__( self._machobj = Machines(machine=machine_name) - if get_model() == "e3sm": + self._config = Config.instance() + + if self._config.calculate_mode_build_cost: # Current build system is unlikely to be able to productively use more than 16 cores self._model_build_cost = min( 16, int((self._machobj.get_value("GMAKE_J") * 2) / 3) + 1 @@ -352,7 +355,7 @@ def __init__( "Use -o to avoid this error".format(existing_baselines), ) - if self._cime_model == "e3sm": + if self._config.sort_tests: _order_tests_by_runtime(test_names, self._baseline_root) # This is the only data that multiple threads will simultaneously access @@ -435,7 +438,7 @@ def __init__( # Setup build groups if single_exe: self._build_groups = [self._tests] - elif self._cime_model == "e3sm": + elif self._config.share_exes: # Any test that's in a shared-enabled suite with other tests should share exes self._build_groups = get_build_groups(self._tests) else: @@ -727,7 +730,7 @@ def _create_newcase_phase(self, test): create_newcase_cmd += " --walltime {}".format(self._walltime) else: # model specific ways of setting time - if self._cime_model == "e3sm": + if self._config.sort_tests: recommended_time = _get_time_est(test, self._baseline_root) if recommended_time is not None: @@ -937,7 +940,7 @@ def _xml_phase(self, test): if self._output_root is None: self._output_root = case.get_value("CIME_OUTPUT_ROOT") # if we are running a single test we don't need sharedlibroot - if len(self._tests) > 1 and self._cime_model != "e3sm": + if len(self._tests) > 1 and self._config.common_sharedlibroot: case.set_value( "SHAREDLIBROOT", os.path.join( @@ -1122,7 +1125,7 @@ def _get_procs_needed(self, test, phase, threads_in_flight=None, no_batch=False) return total_pes elif phase == SHAREDLIB_BUILD_PHASE: - if self._cime_model != "e3sm": + if self._config.serialize_sharedlib_builds: # Will force serialization of sharedlib builds # TODO - instead of serializing, compute all library configs needed and build # them all in parallel @@ -1372,7 +1375,7 @@ def _setup_cs_files(self): os.stat(cs_submit_file).st_mode | stat.S_IXUSR | stat.S_IXGRP, ) - if self._cime_model == "cesm": + if self._config.use_testreporter_template: template_file = os.path.join(template_path, "testreporter.template") template = open(template_file, "r").read() template = template.replace("", get_tools_path()) @@ -1418,8 +1421,10 @@ def run_tests( expect(threading.active_count() == 1, "Leftover threads?") + config = Config.instance() + # Copy TestStatus files to baselines for tests that have already failed. - if get_model() == "cesm": + if config.baseline_store_teststatus: for test in self._tests: status = self._get_test_data(test)[1] if ( diff --git a/CIME/tests/base.py b/CIME/tests/base.py index 110f54011b0..8280736ea1c 100644 --- a/CIME/tests/base.py +++ b/CIME/tests/base.py @@ -11,6 +11,7 @@ import unittest from CIME import utils +from CIME.config import Config from CIME.XML.machines import Machines @@ -149,7 +150,7 @@ def setup_proxy(self): def assert_dashboard_has_build(self, build_name, expected_count=1): # Do not test E3SM dashboard if model is CESM - if utils.get_model() == "e3sm": + if Config.instance().test_mode == "e3sm": time.sleep(10) # Give chance for cdash to update wget_file = tempfile.mktemp() diff --git a/CIME/tests/test_sys_bless_tests_results.py b/CIME/tests/test_sys_bless_tests_results.py index 5184b1f109a..421724da6d0 100644 --- a/CIME/tests/test_sys_bless_tests_results.py +++ b/CIME/tests/test_sys_bless_tests_results.py @@ -6,8 +6,11 @@ import stat from CIME import utils +from CIME.config import Config from CIME.tests import base +config = Config.instance() + class TestBlessTestResults(base.BaseTestCase): def setUp(self): @@ -40,7 +43,7 @@ def test_bless_test_results(self): # Generate some baselines for test_name in test_names: - if utils.get_model() == "e3sm": + if config.create_test_flag_mode == "e3sm": genargs = ["-g", "-o", "-b", self._baseline_name, test_name] compargs = ["-c", "-b", self._baseline_name, test_name] else: @@ -104,7 +107,7 @@ def test_rebless_namelist(self): if self.NO_FORTRAN_RUN: self.skipTest("Skipping fortran test") test_to_change = "TESTRUNPASS_P1.f19_g16_rx1.A" - if utils.get_model() == "e3sm": + if config.create_test_flag_mode == "e3sm": genargs = ["-g", "-o", "-b", self._baseline_name, "cime_test_only_pass"] compargs = ["-c", "-b", self._baseline_name, "cime_test_only_pass"] else: diff --git a/CIME/tests/test_sys_cime_case.py b/CIME/tests/test_sys_cime_case.py index 5c583677cec..c783235c522 100644 --- a/CIME/tests/test_sys_cime_case.py +++ b/CIME/tests/test_sys_cime_case.py @@ -8,10 +8,13 @@ import time from CIME import utils +from CIME.config import Config from CIME.tests import base from CIME.case.case import Case from CIME.XML.env_run import EnvRun +config = Config.instance() + class TestCimeCase(base.BaseTestCase): def test_cime_case(self): @@ -61,7 +64,7 @@ def _batch_test_fixture(self, testcase_name): args = "--case {name} --script-root {testdir} --compset X --res f19_g16 --handle-preexisting-dirs=r --output-root {testdir}".format( name=testcase_name, testdir=testdir ) - if utils.get_model() == "cesm": + if config.allow_unsupported: args += " --run-unsupported" self.run_cmd_assert_result( @@ -310,7 +313,7 @@ def test_cime_case_xmlchange_append(self): self.assertEqual(result, "-opt1 -opt2") def test_cime_case_test_walltime_mgmt_1(self): - if utils.get_model() != "e3sm": + if config.test_mode == "cesm": self.skipTest("Skipping walltime test. Depends on E3SM batch settings") test_name = "ERS.f19_g16_rx1.A" @@ -332,7 +335,7 @@ def test_cime_case_test_walltime_mgmt_1(self): self.assertEqual(result, "biggpu") def test_cime_case_test_walltime_mgmt_2(self): - if utils.get_model() != "e3sm": + if config.test_mode == "cesm": self.skipTest("Skipping walltime test. Depends on E3SM batch settings") test_name = "ERS_P64.f19_g16_rx1.A" @@ -354,7 +357,7 @@ def test_cime_case_test_walltime_mgmt_2(self): self.assertEqual(result, "biggpu") def test_cime_case_test_walltime_mgmt_3(self): - if utils.get_model() != "e3sm": + if config.test_mode == "cesm": self.skipTest("Skipping walltime test. Depends on E3SM batch settings") test_name = "ERS_P64.f19_g16_rx1.A" @@ -382,7 +385,7 @@ def test_cime_case_test_walltime_mgmt_3(self): self.assertEqual(result, "biggpu") # Not smart enough to select faster queue def test_cime_case_test_walltime_mgmt_4(self): - if utils.get_model() != "e3sm": + if config.test_mode == "cesm": self.skipTest("Skipping walltime test. Depends on E3SM batch settings") test_name = "ERS_P1.f19_g16_rx1.A" @@ -410,7 +413,7 @@ def test_cime_case_test_walltime_mgmt_4(self): self.assertEqual(result, "biggpu") def test_cime_case_test_walltime_mgmt_5(self): - if utils.get_model() != "e3sm": + if config.test_mode == "cesm": self.skipTest("Skipping walltime test. Depends on E3SM batch settings") test_name = "ERS_P1.f19_g16_rx1.A" @@ -501,7 +504,7 @@ def test_cime_case_test_walltime_mgmt_7(self): self.assertEqual(result, "421:32:11") def test_cime_case_test_walltime_mgmt_8(self): - if utils.get_model() != "e3sm": + if config.test_mode == "cesm": self.skipTest("Skipping walltime test. Depends on E3SM batch settings") test_name = "SMS_P25600.f19_g16_rx1.A" @@ -533,10 +536,7 @@ def test_cime_case_test_walltime_mgmt_8(self): def test_cime_case_test_custom_project(self): test_name = "ERS_P1.f19_g16_rx1.A" # have to use a machine both models know and one that doesn't put PROJECT in any key paths - if utils.get_model() == "e3sm": - machine = "mappy" - else: - machine = "melvin" + machine = config.test_custom_project_machine compiler = "gnu" casedir = self._create_test( [ diff --git a/CIME/tests/test_sys_create_newcase.py b/CIME/tests/test_sys_create_newcase.py index 4ae53ac03ed..6dc5fc518d3 100644 --- a/CIME/tests/test_sys_create_newcase.py +++ b/CIME/tests/test_sys_create_newcase.py @@ -7,10 +7,13 @@ import sys from CIME import utils +from CIME.config import Config from CIME.tests import base from CIME.case.case import Case from CIME.build import CmakeTmpBuildDir +config = Config.instance() + class TestCreateNewcase(base.BaseTestCase): @classmethod @@ -34,7 +37,7 @@ def test_a_createnewcase(self): testdir, cls._testroot, ) - if utils.get_model() == "cesm": + if config.allow_unsupported: args += " --run-unsupported" if self.TEST_COMPILER is not None: args = args + " --compiler %s" % self.TEST_COMPILER @@ -148,7 +151,7 @@ def test_b_user_mods(self): " --case %s --compset X --user-mods-dir %s --output-root %s --handle-preexisting-dirs=r" % (testdir, user_mods_dir, cls._testroot) ) - if utils.get_model() == "cesm": + if config.allow_unsupported: args += " --run-unsupported" if self.TEST_COMPILER is not None: args = args + " --compiler %s" % self.TEST_COMPILER @@ -322,7 +325,7 @@ def test_f_createnewcase_with_user_compset(self): cls._testdirs.append(testdir) - if utils.get_model() == "cesm": + if config.test_mode == "cesm": if utils.get_cime_default_driver() == "nuopc": pesfile = os.path.join( utils.get_src_root(), @@ -349,7 +352,7 @@ def test_f_createnewcase_with_user_compset(self): "--case %s --compset 2000_SATM_XLND_SICE_SOCN_XROF_XGLC_SWAV --pesfile %s --res f19_g16 --output-root %s --handle-preexisting-dirs=r" % (testdir, pesfile, cls._testroot) ) - if utils.get_model() == "cesm": + if config.allow_unsupported: args += " --run-unsupported" if self.TEST_COMPILER is not None: args += " --compiler %s" % self.TEST_COMPILER @@ -382,7 +385,7 @@ def test_g_createnewcase_with_user_compset_and_env_mach_pes(self): "--case %s --compset 2000_SATM_XLND_SICE_SOCN_XROF_XGLC_SWAV --pesfile %s --res f19_g16 --output-root %s --handle-preexisting-dirs=r" % (testdir, pesfile, cls._testroot) ) - if utils.get_model() == "cesm": + if config.allow_unsupported: args += " --run-unsupported" if self.TEST_COMPILER is not None: args += " --compiler %s" % self.TEST_COMPILER @@ -419,7 +422,7 @@ def test_h_primary_component(self): " --case CreateNewcaseTest --script-root %s --compset X --output-root %s --handle-preexisting-dirs u" % (testdir, cls._testroot) ) - if utils.get_model() == "cesm": + if config.allow_unsupported: args += " --run-unsupported" if self.TEST_COMPILER is not None: args += " --compiler %s" % self.TEST_COMPILER @@ -541,7 +544,7 @@ def test_j_createnewcase_user_compset_vs_alias(self): args += " --res f19_g17 " else: args += " --res f19_g16 " - if utils.get_model() == "cesm": + if config.allow_unsupported: args += " --run-unsupported" if self.TEST_COMPILER is not None: args += " --compiler %s" % self.TEST_COMPILER @@ -582,7 +585,7 @@ def test_j_createnewcase_user_compset_vs_alias(self): args += " --res f19_g17 " else: args += " --res f19_g16 " - if utils.get_model() == "cesm": + if config.allow_unsupported: args += " --run-unsupported" if self.TEST_COMPILER is not None: args += " --compiler %s" % self.TEST_COMPILER @@ -698,7 +701,7 @@ def test_ka_createnewcase_extra_machines_dir(self): extra_machines_dir=extra_machines_dir, ) ) - if utils.get_model() == "cesm": + if config.allow_unsupported: args += " --run-unsupported" if utils.get_cime_default_driver() == "nuopc": @@ -731,18 +734,20 @@ def test_ka_createnewcase_extra_machines_dir(self): def test_m_createnewcase_alternate_drivers(self): # Test that case.setup runs for nuopc and moab drivers cls = self.__class__ - model = utils.get_model() - for driver in ("nuopc", "moab"): + + # TODO refactor + if config.test_mode == "cesm": + alternative_driver = ("nuopc",) + else: + alternative_driver = ("moab",) + + for driver in alternative_driver: if not os.path.exists( os.path.join(utils.get_cime_root(), "src", "drivers", driver) ): self.skipTest( "Skipping driver test for {}, driver not found".format(driver) ) - if (model == "cesm" and driver == "moab") or ( - model == "e3sm" and driver == "nuopc" - ): - continue testdir = os.path.join(cls._testroot, "testcreatenewcase.{}".format(driver)) if os.path.exists(testdir): @@ -750,7 +755,7 @@ def test_m_createnewcase_alternate_drivers(self): args = " --driver {} --case {} --compset X --res f19_g16 --output-root {} --handle-preexisting-dirs=r".format( driver, testdir, cls._testroot ) - if model == "cesm": + if config.allow_unsupported: args += " --run-unsupported" if self.TEST_COMPILER is not None: args = args + " --compiler %s" % self.TEST_COMPILER @@ -777,7 +782,6 @@ def test_m_createnewcase_alternate_drivers(self): def test_n_createnewcase_bad_compset(self): cls = self.__class__ - model = utils.get_model() testdir = os.path.join(cls._testroot, "testcreatenewcase_bad_compset") if os.path.exists(testdir): @@ -786,7 +790,7 @@ def test_n_createnewcase_bad_compset(self): " --case %s --compset InvalidCompsetName --output-root %s --handle-preexisting-dirs=r " % (testdir, cls._testroot) ) - if model == "cesm": + if config.allow_unsupported: args += " --run-unsupported" if self.TEST_COMPILER is not None: args = args + " --compiler %s" % self.TEST_COMPILER diff --git a/CIME/tests/test_sys_grid_generation.py b/CIME/tests/test_sys_grid_generation.py index 8300b98a87d..a16ea1f2e52 100644 --- a/CIME/tests/test_sys_grid_generation.py +++ b/CIME/tests/test_sys_grid_generation.py @@ -5,6 +5,7 @@ import sys from CIME import utils +from CIME.config import Config from CIME.tests import base @@ -16,7 +17,7 @@ def setUpClass(cls): cls._testdirs = [] def test_gen_domain(self): - if utils.get_model() != "e3sm": + if Config.instance().test_mode == "cesm": self.skipTest("Skipping gen_domain test. Depends on E3SM tools") cime_root = utils.get_cime_root() inputdata = self.MACHINE.get_value("DIN_LOC_ROOT") diff --git a/CIME/tests/test_sys_jenkins_generic_job.py b/CIME/tests/test_sys_jenkins_generic_job.py index c2c9acd79fc..a600d3b22fa 100644 --- a/CIME/tests/test_sys_jenkins_generic_job.py +++ b/CIME/tests/test_sys_jenkins_generic_job.py @@ -9,12 +9,15 @@ from CIME import get_tests from CIME import utils +from CIME.config import Config from CIME.tests import base +config = Config.instance() + class TestJenkinsGenericJob(base.BaseTestCase): def setUp(self): - if utils.get_model() != "e3sm": + if config.test_mode == "cesm": self.skipTest("Skipping Jenkins tests. E3SM feature") super().setUp() @@ -39,7 +42,7 @@ def simple_test(self, expect_works, extra_args, build_name=None): extra_args += " --no-batch" # Need these flags to test dashboard if e3sm - if utils.get_model() == "e3sm" and build_name is not None: + if config.test_mode == "e3sm" and build_name is not None: extra_args += ( " -p ACME_test --submit-to-cdash --cdash-build-group=Nightly -c %s" % build_name diff --git a/CIME/tests/test_sys_manage_and_query.py b/CIME/tests/test_sys_manage_and_query.py index 2d0350e571c..0479704f222 100644 --- a/CIME/tests/test_sys_manage_and_query.py +++ b/CIME/tests/test_sys_manage_and_query.py @@ -1,6 +1,7 @@ #!/usr/bin/env python3 from CIME import utils +from CIME.config import Config from CIME.tests import base from CIME.XML.files import Files @@ -9,7 +10,7 @@ class TestManageAndQuery(base.BaseTestCase): """Tests various scripts to manage and query xml files""" def setUp(self): - if utils.get_model() == "e3sm": + if Config.instance().test_mode == "e3sm": self.skipTest("Skipping XML test management tests. E3SM does not use this.") super().setUp() diff --git a/CIME/tests/test_sys_save_timings.py b/CIME/tests/test_sys_save_timings.py index d571974c050..ca3e0a61fd0 100644 --- a/CIME/tests/test_sys_save_timings.py +++ b/CIME/tests/test_sys_save_timings.py @@ -6,9 +6,12 @@ from CIME import provenance from CIME import utils +from CIME.config import Config from CIME.tests import base from CIME.case.case import Case +config = Config.instance() + class TestSaveTimings(base.BaseTestCase): def simple_test(self, manual_timing=False): @@ -46,7 +49,7 @@ def simple_test(self, manual_timing=False): self.run_cmd_assert_result( "cd %s && %s/save_provenance postrun" % (casedir, self.TOOLS_DIR) ) - if utils.get_model() == "e3sm": + if config.test_mode == "e3sm": provenance_glob = os.path.join( timing_dir, "performance_archive", @@ -104,7 +107,7 @@ def _record_success( self.assertEqual(exp_last_pass, commit, msg="Should never") def test_success_recording(self): - if utils.get_model() != "e3sm": + if config.test_mode == "e3sm": self.skipTest("Skipping success recording tests. E3SM feature") fake_test1 = "faketest1" diff --git a/CIME/tests/test_sys_single_submit.py b/CIME/tests/test_sys_single_submit.py index 0909e7d83eb..62dfe2014e9 100644 --- a/CIME/tests/test_sys_single_submit.py +++ b/CIME/tests/test_sys_single_submit.py @@ -1,6 +1,7 @@ #!/usr/bin/env python3 from CIME import utils +from CIME.config import Config from CIME.tests import base @@ -9,7 +10,7 @@ def test_single_submit(self): # Skip unless on a batch system and users did not select no-batch if not self._hasbatch: self.skipTest("Skipping single submit. Not valid without batch") - if utils.get_model() != "e3sm": + if Config.instance().test_mode == "cesm": self.skipTest("Skipping single submit. E3SM experimental feature") if self._machine not in ["sandiatoss3"]: self.skipTest("Skipping single submit. Only works on sandiatoss3") diff --git a/CIME/tests/test_sys_test_scheduler.py b/CIME/tests/test_sys_test_scheduler.py index fc1eb9daf06..2e56f96a786 100755 --- a/CIME/tests/test_sys_test_scheduler.py +++ b/CIME/tests/test_sys_test_scheduler.py @@ -10,13 +10,14 @@ from CIME import utils from CIME import test_status from CIME import test_scheduler +from CIME.config import Config from CIME.tests import base class TestTestScheduler(base.BaseTestCase): @mock.patch("time.strftime", return_value="00:00:00") def test_chksum(self, strftime): # pylint: disable=unused-argument - if utils.get_model() != "cesm": + if Config.instance().test_mode == "e3sm": self.skipTest("Skipping chksum test. Depends on CESM settings") ts = test_scheduler.TestScheduler( diff --git a/CIME/tests/test_sys_wait_for_tests.py b/CIME/tests/test_sys_wait_for_tests.py index d6647571d78..3afdd9877f4 100644 --- a/CIME/tests/test_sys_wait_for_tests.py +++ b/CIME/tests/test_sys_wait_for_tests.py @@ -9,9 +9,12 @@ from CIME import utils from CIME import test_status +from CIME.config import Config from CIME.tests import base from CIME.tests import utils as test_utils +config = Config.instance() + class TestWaitForTests(base.BaseTestCase): def setUp(self): @@ -113,7 +116,7 @@ def tearDown(self): def simple_test(self, testdir, expected_results, extra_args="", build_name=None): # Need these flags to test dashboard if e3sm - if utils.get_model() == "e3sm" and build_name is not None: + if config.create_test_flag_mode == "e3sm" and build_name is not None: extra_args += " -b %s" % build_name expected_stat = 0 @@ -297,7 +300,7 @@ def test_wait_for_test_cdash_kill(self): self.assert_dashboard_has_build(build_name) - if utils.get_model() == "e3sm": + if config.test_mode == "e3sm": cdash_result_dir = os.path.join(self._testdir_unfinished, "Testing") tag_file = os.path.join(cdash_result_dir, "TAG") self.assertTrue(os.path.isdir(cdash_result_dir)) diff --git a/CIME/tests/test_unit_config.py b/CIME/tests/test_unit_config.py new file mode 100644 index 00000000000..62fec32bbf5 --- /dev/null +++ b/CIME/tests/test_unit_config.py @@ -0,0 +1,136 @@ +import os +import unittest +import tempfile + +from CIME.config import Config + + +class TestConfig(unittest.TestCase): + def test_class_external(self): + with tempfile.TemporaryDirectory() as tempdir: + complex_file = os.path.join(tempdir, "01_complex.py") + + with open(complex_file, "w") as fd: + fd.write( + """ +class TestComplex: + def do_something(self): + print("Something complex") + """ + ) + + test_file = os.path.join(tempdir, "02_test.py") + + with open(test_file, "w") as fd: + fd.write( + """ +from CIME.customize import TestComplex + +use_feature1 = True +use_feature2 = False + +def prerun_provenance(case, **kwargs): + print("prerun_provenance") + + external = TestComplex() + + external.do_something() + + return True + """ + ) + + config = Config.load(tempdir) + + assert config.use_feature1 + assert not config.use_feature2 + assert config.prerun_provenance + assert config.prerun_provenance("test") + + with self.assertRaises(AttributeError): + config.postrun_provenance("test") + + def test_class(self): + with tempfile.TemporaryDirectory() as tempdir: + test_file = os.path.join(tempdir, "test.py") + + with open(test_file, "w") as fd: + fd.write( + """ +use_feature1 = True +use_feature2 = False + +class TestComplex: + def do_something(self): + print("Something complex") + +def prerun_provenance(case, **kwargs): + print("prerun_provenance") + + external = TestComplex() + + external.do_something() + + return True + """ + ) + + config = Config.load(tempdir) + + assert config.use_feature1 + assert not config.use_feature2 + assert config.prerun_provenance + assert config.prerun_provenance("test") + + with self.assertRaises(AttributeError): + config.postrun_provenance("test") + + def test_load(self): + with tempfile.TemporaryDirectory() as tempdir: + test_file = os.path.join(tempdir, "test.py") + + with open(test_file, "w") as fd: + fd.write( + """ +use_feature1 = True +use_feature2 = False + +def prerun_provenance(case, **kwargs): + print("prerun_provenance") + + return True + """ + ) + + config = Config.load(tempdir) + + assert config.use_feature1 + assert not config.use_feature2 + assert config.prerun_provenance + assert config.prerun_provenance("test") + + with self.assertRaises(AttributeError): + config.postrun_provenance("test") + + def test_overwrite(self): + with tempfile.TemporaryDirectory() as tempdir: + test_file = os.path.join(tempdir, "test.py") + + with open(test_file, "w") as fd: + fd.write( + """ +use_feature1 = True +use_feature2 = False + +def prerun_provenance(case, **kwargs): + print("prerun_provenance") + + return True + """ + ) + + Config.use_feature1 = False + + config = Config.load(tempdir) + + assert config.use_feature1 diff --git a/CIME/tests/test_unit_provenance.py b/CIME/tests/test_unit_provenance.py deleted file mode 100755 index 8ee764f4629..00000000000 --- a/CIME/tests/test_unit_provenance.py +++ /dev/null @@ -1,216 +0,0 @@ -#!/usr/bin/env python3 - -import os -import sys -import tempfile -import unittest -from unittest import mock - -from CIME import provenance -from CIME import utils - -# pylint: disable=protected-access -class TestProvenance(unittest.TestCase): - def test_parse_dot_git_path_error(self): - with self.assertRaises(utils.CIMEError): - provenance._parse_dot_git_path("/src/CIME") - - def test_parse_dot_git_path(self): - value = provenance._parse_dot_git_path("/src/CIME/.git/worktrees/test") - - assert value == "/src/CIME/.git" - - def test_read_gitdir(self): - with tempfile.TemporaryDirectory() as tempdir: - dot_git_path = os.path.join(tempdir, ".git") - - with open(dot_git_path, "w") as fd: - fd.write("gitdir: /src/CIME/.git/worktrees/test") - - value = provenance._read_gitdir(dot_git_path) - - assert value == "/src/CIME/.git/worktrees/test" - - with open(dot_git_path, "w") as fd: - fd.write("gitdir:/src/CIME/.git/worktrees/test") - - value = provenance._read_gitdir(dot_git_path) - - assert value == "/src/CIME/.git/worktrees/test" - - def test_read_gitdir_not_file(self): - with tempfile.TemporaryDirectory() as tempdir: - dot_git_path = os.path.join(tempdir, ".git") - - os.makedirs(dot_git_path) - - with self.assertRaises(utils.CIMEError): - provenance._read_gitdir(dot_git_path) - - def test_read_gitdir_bad_contents(self): - with tempfile.TemporaryDirectory() as tempdir: - dot_git_path = os.path.join(tempdir, ".git") - - with open(dot_git_path, "w") as fd: - fd.write("some value: /src/CIME/.git/worktrees/test") - - with self.assertRaises(utils.CIMEError): - provenance._read_gitdir(dot_git_path) - - def test_find_git_root(self): - with tempfile.TemporaryDirectory() as tempdir: - os.makedirs(os.path.join(tempdir, ".git")) - - value = provenance._find_git_root(tempdir) - - assert value == f"{tempdir}/.git" - - def test_find_git_root_does_not_exist(self): - with tempfile.TemporaryDirectory() as tempdir: - with self.assertRaises(utils.CIMEError): - provenance._find_git_root(tempdir) - - def test_find_git_root_submodule(self): - with tempfile.TemporaryDirectory() as tempdir: - cime_path = os.path.join(tempdir, "cime") - - os.makedirs(cime_path) - - cime_git_dot_file = os.path.join(cime_path, ".git") - - with open(cime_git_dot_file, "w") as fd: - fd.write(f"gitdir: {tempdir}/.git/modules/cime") - - temp_dot_git_path = os.path.join(tempdir, ".git", "modules", "cime") - - os.makedirs(temp_dot_git_path) - - temp_config = os.path.join(temp_dot_git_path, "config") - - with open(temp_config, "w") as fd: - fd.write("") - - value = provenance._find_git_root(cime_path) - - assert value == f"{tempdir}/.git/modules/cime" - - # relative path - with open(cime_git_dot_file, "w") as fd: - fd.write(f"gitdir: ../.git/modules/cime") - - value = provenance._find_git_root(cime_path) - - assert value == f"{tempdir}/.git/modules/cime" - - def test_find_git_root_worktree(self): - with tempfile.TemporaryDirectory() as tempdir: - git_dot_file = os.path.join(tempdir, ".git") - - with open(git_dot_file, "w") as fd: - fd.write("gitdir: /src/CIME/.git/worktrees/test") - - value = provenance._find_git_root(tempdir) - - assert value == "/src/CIME/.git" - - def test_find_git_root_worktree_bad_contents(self): - with tempfile.TemporaryDirectory() as tempdir: - with open(os.path.join(tempdir, ".git"), "w") as fd: - fd.write("some value: /src/CIME/.git/worktrees/test") - - with self.assertRaises(utils.CIMEError): - provenance._find_git_root(tempdir) - - @mock.patch("CIME.provenance.run_cmd") - def test_run_git_cmd_recursively(self, run_cmd): - run_cmd.return_value = (0, "data", None) - - with mock.patch("CIME.provenance.open", mock.mock_open()) as m: - provenance._run_git_cmd_recursively( - "status", "/srcroot", "/output.txt" - ) # pylint: disable=protected-access - - m.assert_called_with("/output.txt", "w") - - write = m.return_value.__enter__.return_value.write - - write.assert_any_call("data\n\n") - write.assert_any_call("data\n") - - run_cmd.assert_any_call("git status", from_dir="/srcroot") - run_cmd.assert_any_call( - 'git submodule foreach --recursive "git status; echo"', from_dir="/srcroot" - ) - - @mock.patch("CIME.provenance.run_cmd") - def test_run_git_cmd_recursively_error(self, run_cmd): - run_cmd.return_value = (1, "data", "error") - - with mock.patch("CIME.provenance.open", mock.mock_open()) as m: - provenance._run_git_cmd_recursively( - "status", "/srcroot", "/output.txt" - ) # pylint: disable=protected-access - - m.assert_called_with("/output.txt", "w") - - write = m.return_value.__enter__.return_value.write - - write.assert_any_call("error\n\n") - write.assert_any_call("error\n") - - run_cmd.assert_any_call("git status", from_dir="/srcroot") - run_cmd.assert_any_call( - 'git submodule foreach --recursive "git status; echo"', from_dir="/srcroot" - ) - - @mock.patch("CIME.provenance.safe_copy") - @mock.patch("CIME.provenance.run_cmd") - def test_record_git_provenance(self, run_cmd, safe_copy): - run_cmd.return_value = (0, "data", None) - - with mock.patch("CIME.provenance.open", mock.mock_open()) as m: - with tempfile.TemporaryDirectory() as tempdir: - os.makedirs(os.path.join(tempdir, ".git")) - - provenance._record_git_provenance( - tempdir, "/output", "5" - ) # pylint: disable=protected-access - - m.assert_any_call("/output/GIT_STATUS.5", "w") - m.assert_any_call("/output/GIT_DIFF.5", "w") - m.assert_any_call("/output/GIT_LOG.5", "w") - m.assert_any_call("/output/GIT_REMOTE.5", "w") - - write = m.return_value.__enter__.return_value.write - - write.assert_any_call("data\n\n") - write.assert_any_call("data\n") - - run_cmd.assert_any_call("git status", from_dir=tempdir) - run_cmd.assert_any_call( - 'git submodule foreach --recursive "git status; echo"', from_dir=tempdir - ) - run_cmd.assert_any_call("git diff", from_dir=tempdir) - run_cmd.assert_any_call( - 'git submodule foreach --recursive "git diff; echo"', from_dir=tempdir - ) - run_cmd.assert_any_call( - "git log --first-parent --pretty=oneline -n 5", from_dir=tempdir - ) - run_cmd.assert_any_call( - 'git submodule foreach --recursive "git log --first-parent' - ' --pretty=oneline -n 5; echo"', - from_dir=tempdir, - ) - run_cmd.assert_any_call("git remote -v", from_dir=tempdir) - run_cmd.assert_any_call( - 'git submodule foreach --recursive "git remote -v; echo"', from_dir=tempdir - ) - - safe_copy.assert_any_call( - f"{tempdir}/.git/config", "/output/GIT_CONFIG.5", preserve_meta=False - ) - - -if __name__ == "__main__": - unittest.main() diff --git a/CIME/utils.py b/CIME/utils.py index 309ea650631..20828361cbd 100644 --- a/CIME/utils.py +++ b/CIME/utils.py @@ -453,14 +453,16 @@ def get_cime_default_driver(): logger.debug( "Setting CIME_driver={} from ~/.cime/config".format(driver) ) + + from CIME.config import Config + + config = Config.instance() + if not driver: - model = get_model() - if model == "ufs" or model == "cesm": - driver = "nuopc" - else: - driver = "mct" + driver = config.driver_default + expect( - driver in ("mct", "nuopc", "moab"), + driver in config.driver_choices, "Attempt to set invalid driver {}".format(driver), ) return driver @@ -2475,7 +2477,11 @@ def run_and_log_case_status( def _check_for_invalid_args(args): - if get_model() != "e3sm": + # Prevent circular import + from CIME.config import Config + + # TODO Is this really model specific + if Config.instance().check_invalid_args: for arg in args: # if arg contains a space then it was originally quoted and we can ignore it here. if " " in arg or arg.startswith("--"): diff --git a/conftest.py b/conftest.py index dce9e8b6cbb..fcee11dfa00 100644 --- a/conftest.py +++ b/conftest.py @@ -7,6 +7,7 @@ import pytest from CIME import utils +from CIME.config import Config from CIME.tests import scripts_regression_tests os.environ["CIME_GLOBAL_WALLTIME"] = "0:05:00" @@ -31,6 +32,17 @@ def pytest_configure(config): scripts_regression_tests.configure_tests(**kwargs) -@pytest.fixture(scope="session", autouse=True) +@pytest.fixture(scope="module", autouse=True) def setup(pytestconfig): + # ensure we start from CIMEROOT for each module os.chdir(CIMEROOT) + + srcroot = utils.get_src_root() + + customize_path = os.path.join(srcroot, "cime_config", "customize") + + if os.path.exists(customize_path): + Config.instance().load(customize_path) + + # ensure GLOABL is reset + utils.GLOBAL = {} diff --git a/doc/Makefile b/doc/Makefile index ee5f25834b1..93d79cf7e08 100644 --- a/doc/Makefile +++ b/doc/Makefile @@ -8,7 +8,6 @@ SPHINXPROJ = on SOURCEDIR = source BUILDDIR = build SPHINXAPI = sphinx-apidoc -SCRIPTSDIR = ../scripts # Put it first so that "make" without argument is like "make help". help: @@ -20,10 +19,10 @@ api: CIME_api Tools_api Tools_user exit 0 CIME_api: - @$(SPHINXAPI) --force -o $(SOURCEDIR)/$@ $(SCRIPTSDIR)/lib/CIME + @$(SPHINXAPI) --force -o $(SOURCEDIR)/$@ ../CIME Tools_api: - @$(SPHINXAPI) --force -o $(SOURCEDIR)/$@ $(SCRIPTSDIR)/Tools + @$(SPHINXAPI) --force -o $(SOURCEDIR)/$@ ../CIME/Tools Tools_user: rm -f $(SOURCEDIR)/$@/*.rst diff --git a/doc/source/Tools_user/index.rst.template b/doc/source/Tools_user/index.rst.template index 06d03fddbb1..e7c67e2a107 100644 --- a/doc/source/Tools_user/index.rst.template +++ b/doc/source/Tools_user/index.rst.template @@ -10,3 +10,4 @@ and **case.setup**. .. toctree:: :maxdepth: 1 + diff --git a/doc/source/conf.py b/doc/source/conf.py index ddc2ab3f09f..ada46cc4f62 100644 --- a/doc/source/conf.py +++ b/doc/source/conf.py @@ -23,9 +23,9 @@ # pip install git+https://github.com/esmci/sphinx_rtd_theme.git@version-dropdown-with-fixes import sphinx_rtd_theme -sys.path.insert(0, os.path.abspath("../../scripts/lib")) +sys.path.insert(0, os.path.abspath("../../")) sys.path.insert(1, os.path.abspath("../../scripts")) -sys.path.insert(2, os.path.abspath("../../scripts/Tools")) +sys.path.insert(2, os.path.abspath("../../CIME")) # -- General configuration ------------------------------------------------ diff --git a/doc/source/users_guide/cime-customize.rst b/doc/source/users_guide/cime-customize.rst new file mode 100644 index 00000000000..ed90e21472a --- /dev/null +++ b/doc/source/users_guide/cime-customize.rst @@ -0,0 +1,77 @@ +.. _customizing-cime: + +=========================== +CIME config and hooks +=========================== + +CIME provides the ability to define model specific config and hooks. + +The config alters CIME's runtime and the hooks are triggered during their event. + +----------------------------------- +How does CIME load customizations? +----------------------------------- + +CIME will search ``cime_config/customize`` and load any python found under this directory or it's children. + +Any variables, functions or classes loaded are available from the ``CIME.customize`` module. + +--------------------------- +CIME config +--------------------------- + +Available config and descriptions. + +================================= ======================= ===== ================================================================================================================================================================================================================================ +Variable Default Type Description +================================= ======================= ===== ================================================================================================================================================================================================================================ +additional_archive_components ('drv', 'dart') tuple Additional components to archive. +allow_unsupported True bool If set to `True` then unsupported compsets and resolutions are allowed. +baseline_store_teststatus True bool If set to `True` and GENERATE_BASELINE is set then a teststatus.log is created in the case's baseline. +build_cime_component_lib True bool If set to `True` then `Filepath`, `CIME_cppdefs` and `CCSM_cppdefs` directories are copied from CASEBUILD directory to BUILDROOT in order to build CIME's internal components. +build_model_use_cmake False bool If set to `True` the model is built using using CMake otherwise Make is used. +calculate_mode_build_cost False bool If set to `True` then the TestScheduler will set the number of processors for building the model to min(16, (($GMAKE_J * 2) / 3) + 1) otherwise it's set to 4. +case_setup_generate_namelist False bool If set to `True` and case is a test then namelists are created during `case.setup`. +check_invalid_args True bool If set to `True` then script arguments are checked for being valid. +check_machine_name_from_test_name True bool If set to `True` then the TestScheduler will use testlists to parse for a list of tests. +common_sharedlibroot True bool If set to `True` then SHAREDLIBROOT is set for the case and SystemTests will only build the shared libs once. +copy_cesm_tools True bool If set to `True` then CESM specific tools are copied into the case directory. +copy_cism_source_mods True bool If set to `True` then `$CASEROOT/SourceMods/src.cism/source_cism` is created and a README is written to directory. +copy_e3sm_tools False bool If set to `True` then E3SM specific tools are copied into the case directory. +create_bless_log False bool If set to `True` and comparing test to baselines the most recent bless is added to comments. +create_test_flag_mode cesm str Sets the flag mode for the `create_test` script. When set to `cesm`, the `-c` flag will compare baselines against a give directory. +default_short_term_archiving True bool If set to `True` and the case is not a test then DOUT_S is set to True and TIMER_LEVEL is set to 4. +driver_choices ('mct', 'nuopc') tuple Sets the available driver choices for the model. +driver_default nuopc str Sets the default driver for the model. +enable_smp True bool If set to `True` then `SMP=` is added to model compile command. +gpus_use_set_device_rank True bool If set to `True` and NGPUS_PER_NODE > 0 then `$RUNDIR/set_device_rank.sh` is appended when the MPI run command is generated. +make_case_run_batch_script False bool If set to `True` and case is not a test then `case.run.sh` is created in case directory from `$MACHDIR/template.case.run.sh`. +mct_path {srcroot}/libraries/mct str Sets the path to the mct library. +serialize_sharedlib_builds True bool If set to `True` then the TestScheduler will use `proc_pool + 1` processors to build shared libraries otherwise a single processor is used. +set_comp_root_dir_cpl True bool If set to `True` then COMP_ROOT_DIR_CPL is set for the case. +share_exes False bool If set to `True` then the TestScheduler will share exes between tests. +shared_clm_component True bool If set to `True` and then the `clm` land component is built as a shared lib. +sort_tests False bool If set to `True` then the TestScheduler will sort tests by runtime. +test_custom_project_machine melvin str Sets the machine name to use when testing a machine with no PROJECT. +test_mode cesm str Sets the testing mode, this changes various configuration for CIME's unit and system tests. +ufs_alternative_config False bool If set to `True` and UFS_DRIVER is set to `nems` then model config dir is set to `$CIMEROOT/../src/model/NEMS/cime/cime_config`. +use_kokkos False bool If set to `True` and CAM_TARGET is `preqx_kokkos`, `theta-l` or `theta-l_kokkos` then kokkos is built with the shared libs. +use_nems_comp_root_dir False bool If set to `True` then COMP_ROOT_DIR_CPL is set using UFS_DRIVER if defined. +use_testreporter_template True bool If set to `True` then the TestScheduler will create `testreporter` in $CIME_OUTPUT_ROOT. +verbose_run_phase False bool If set to `True` then after a SystemTests successful run phase the elapsed time is recorded to BASELINE_ROOT, on a failure the test is checked against the previous run and potential breaking merges are listed in the testlog. +xml_component_key COMP_ROOT_DIR_{} str The string template used as the key to query the XML system to find a components root directory e.g. the template `COMP_ROOT_DIR_{}` and component `LND` becomes `COMP_ROOT_DIR_LND`. +================================= ======================= ===== ================================================================================================================================================================================================================================ + +--------------------------- +CIME hooks +--------------------------- + +Available hooks and descriptions. + +======================================= ================================= +Function Description +======================================= ================================= +``save_build_provenance(case, lid)`` Called after the model is built. +``save_prerun_provenance(case, lid)`` Called before the model is run. +``save_postrun_provenance(case, lid)`` Called after the model is run. +======================================= ================================= diff --git a/doc/source/users_guide/index.rst b/doc/source/users_guide/index.rst index f86173b5795..9df9b084490 100644 --- a/doc/source/users_guide/index.rst +++ b/doc/source/users_guide/index.rst @@ -21,6 +21,7 @@ Case Control System Part 1: Basic Usage cloning-a-case.rst cime-change-namelist.rst cime-config.rst + cime-customize.rst troubleshooting.rst .. _users-guide2: diff --git a/doc/tools_autodoc.cfg b/doc/tools_autodoc.cfg index 9472086e724..c0883b64e88 100644 --- a/doc/tools_autodoc.cfg +++ b/doc/tools_autodoc.cfg @@ -1,5 +1,5 @@ [tools] -tools_dir: ../scripts/Tools +tools_dir: ../CIME/Tools exclude_files: __init__.py load.awk standard_script_setup.py Makefile exclude_ext: ~ pyc exclude_prefix: JENKINS_ diff --git a/doc/tools_autodoc.py b/doc/tools_autodoc.py index 654dc262e50..c2a420751f6 100755 --- a/doc/tools_autodoc.py +++ b/doc/tools_autodoc.py @@ -41,7 +41,7 @@ $tool_name #################################################### -**$tool_name** is a script in CIMEROOT/scripts/Tools. +**$tool_name** is a script in CIMEROOT/CIME/Tools. .. toctree:: :maxdepth: 1 diff --git a/docker/config_machines.xml b/docker/config_machines.xml index c6d75be1cae..16a9a3dd460 100644 --- a/docker/config_machines.xml +++ b/docker/config_machines.xml @@ -8,7 +8,9 @@ gnu openmpi - /storage/timings/$CASE + CIME + /storage/timings + CIME /storage/cases /storage/inputdata /storage/inputdata-clmforc @@ -30,8 +32,8 @@ - $CIME_OUTPUT_ROOT/$CASE/run - $CIME_OUTPUT_ROOT/$CASE/bld + $CASEROOT/run + $CASEROOT/bld 1 1 diff --git a/docker/entrypoint.sh b/docker/entrypoint.sh index 50c5c1228b8..d6c966c502b 100755 --- a/docker/entrypoint.sh +++ b/docker/entrypoint.sh @@ -80,7 +80,7 @@ function init_e3sm() { export CIME_MODEL="e3sm" local extras="" - local install_path="${INSTALL_install_path:-/src/E3SM}" + local install_path="${INSTALL_PATH:-/src/E3SM}" local cache_path="${cache_path:-/storage/inputdata}" if [[ ! -e "${install_path}" ]]