Remove provenance/model specific configuration

.pre-commit-config.yaml

@@ -7,7 +7,9 @@ repos:
       - id: check-xml
         files: config/
       - id: end-of-file-fixer
+        exclude: doc/
       - id: trailing-whitespace
+        exclude: doc/
   - repo: https://github.com/psf/black
     rev: 22.3.0
     hooks:
@@ -18,6 +20,6 @@ repos:
     hooks:
       - id: pylint
         args:
-          - --disable=I,C,R,logging-not-lazy,wildcard-import,unused-wildcard-import,fixme,broad-except,bare-except,eval-used,exec-used,global-statement,logging-format-interpolation,no-name-in-module,arguments-renamed,unspecified-encoding,protected-access,import-error
+          - --disable=I,C,R,logging-not-lazy,wildcard-import,unused-wildcard-import,fixme,broad-except,bare-except,eval-used,exec-used,global-statement,logging-format-interpolation,no-name-in-module,arguments-renamed,unspecified-encoding,protected-access,import-error,no-member
         files: CIME
         exclude: CIME/(tests|Tools|code_checker.py)

CIME/SystemTests/system_tests_common.py

@@ -23,6 +23,7 @@
     get_ts_synopsis,
     generate_baseline,
 )
+from CIME.config import Config
 from CIME.provenance import save_test_time, get_test_success
 from CIME.locked_files import LOCKED_DIR, lock_file, is_locked
 import CIME.build as build
@@ -253,7 +254,9 @@ def run(self, skip_pnl=False):
                     RUN_PHASE, status, comments=("time={:d}".format(int(time_taken)))
                 )
 
-            if get_model() == "e3sm":
+            config = Config.instance()
+
+            if config.verbose_run_phase:
                 # If run phase worked, remember the time it took in order to improve later walltime ests
                 baseline_root = self._case.get_value("BASELINE_ROOT")
                 if success:
@@ -320,7 +323,9 @@ def run(self, skip_pnl=False):
                                     )
                                 )
 
-            if get_model() == "cesm" and self._case.get_value("GENERATE_BASELINE"):
+            if config.baseline_store_teststatus and self._case.get_value(
+                "GENERATE_BASELINE"
+            ):
                 baseline_dir = os.path.join(
                     self._case.get_value("BASELINE_ROOT"),
                     self._case.get_value("BASEGEN_CASE"),

CIME/SystemTests/system_tests_compare_n.py

@@ -42,7 +42,7 @@
 from CIME.XML.standard_module_setup import *
 from CIME.SystemTests.system_tests_common import SystemTestsCommon
 from CIME.case import Case
-from CIME.utils import get_model
+from CIME.config import Config
 from CIME.test_status import *
 
 import shutil, os, glob
@@ -182,6 +182,8 @@ def build_phase(self, sharedlib_only=False, model_only=False):
         # with a with statement in all the API entrances in CIME. subsequent cases were
         # created via clone, not a with statement, so it's not in a writeable state,
         # so we need to use a with statement here to put it in a writeable state.
+        config = Config.instance()
+
         for i in range(1, self.N):
             with self._cases[i]:
                 if self._separate_builds:
@@ -193,7 +195,7 @@ def build_phase(self, sharedlib_only=False, model_only=False):
                     # Although we're doing separate builds, it still makes sense
                     # to share the sharedlibroot area with case1 so we can reuse
                     # pieces of the build from there.
-                    if get_model() != "e3sm":
+                    if config.common_sharedlibroot:
                         # We need to turn off this change for E3SM because it breaks
                         # the MPAS build system
                         ## TODO: ^this logic mimics what's done in SystemTestsCompareTwo

CIME/SystemTests/system_tests_compare_two.py

@@ -47,7 +47,7 @@
 from CIME.XML.standard_module_setup import *
 from CIME.SystemTests.system_tests_common import SystemTestsCommon
 from CIME.case import Case
-from CIME.utils import get_model
+from CIME.config import Config
 from CIME.test_status import *
 
 import shutil, os, glob
@@ -217,7 +217,7 @@ def build_phase(self, sharedlib_only=False, model_only=False):
                 # Although we're doing separate builds, it still makes sense
                 # to share the sharedlibroot area with case1 so we can reuse
                 # pieces of the build from there.
-                if get_model() != "e3sm":
+                if Config.instance().common_sharedlibroot:
                     # We need to turn off this change for E3SM because it breaks
                     # the MPAS build system
                     self._case2.set_value(

CIME/Tools/generate_cylc_workflow.py

@@ -3,8 +3,12 @@
 """
 Generates a cylc workflow file for the case.  See https://cylc.github.io for details about cylc
 """
+import os
+import sys
 
-from standard_script_setup import *
+sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..")))
+
+from CIME.Tools.standard_script_setup import *
 
 from CIME.case import Case
 from CIME.utils import expect, transform_vars

CIME/Tools/save_provenance

@@ -7,10 +7,15 @@ This tool provide command-line access to provenance-saving functionality
 from standard_script_setup import *
 
 from CIME.case import Case
+from CIME.config import Config
 from CIME.provenance import *
 from CIME.utils import get_lids
 from CIME.get_timing import get_timing
 
+import logging
+
+logger = logging.getLogger(__name__)
+
 ###############################################################################
 def parse_command_line(args, description):
     ###############################################################################
@@ -57,31 +62,56 @@ def _main_func(description):
     ###############################################################################
     mode, caseroot, lid = parse_command_line(sys.argv, description)
     with Case(caseroot, read_only=False) as case:
+        srcroot = case.get_value("SRCROOT")
+
+        customize_path = os.path.join(srcroot, "cime_config", "customize")
+
+        config = Config.load(customize_path)
+
         if mode == "build":
             expect(
                 False,
                 "Saving build provenance manually is not currently supported "
                 "but it should already always be happening automatically",
             )
-            save_build_provenance(case, lid=lid)
+
+            try:
+                config.save_build_provenance(case, lid=lid)
+            except AttributeError:
+                logger.debug("Could not save build provenance, no handler found")
         elif mode == "prerun":
             expect(lid is not None, "You must provide LID for prerun mode")
-            save_prerun_provenance(case, lid=lid)
+
+            try:
+                config.save_prerun_provenance(case, lid=lid)
+            except AttributeError:
+                logger.debug("Could not save prerun provenance, no handler found")
         elif mode == "postrun":
             expect(lid is None, "Please allow me to autodetect LID")
+
             model = case.get_value("MODEL")
             caseid = case.get_value("CASE")
             case.set_value("SAVE_TIMING", True)
             lids = get_lids(case)
+
             for lid in lids:
                 # call get_timing if needed
                 expected_timing_file = os.path.join(
                     caseroot, "timing", "{}_timing.{}.{}.gz".format(model, caseid, lid)
                 )
+
                 if not os.path.exists(expected_timing_file):
                     get_timing(case, lid)
-                save_prerun_provenance(case, lid=lid)
-                save_postrun_provenance(case, lid=lid)
+
+                try:
+                    config.save_prerun_provenance(case, lid=lid)
+                except AttributeError:
+                    logger.debug("Could not save prerun provenance, no handler found")
+
+                try:
+                    config.save_postrun_provenance(case, lid=lid)
+                except AttributeError:
+                    logger.debug("Could not save postrun provenance, no handler found")
         else:
             expect(False, "Unhandled mode '{}'".format(mode))
 

CIME/Tools/testreporter.py

@@ -3,8 +3,12 @@
 """
 Simple script to populate CESM test database with test results.
 """
+import os
+import sys
 
-from standard_script_setup import *
+sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..")))
+
+from CIME.Tools.standard_script_setup import *
 
 from CIME.XML.env_build import EnvBuild
 from CIME.XML.env_case import EnvCase

CIME/XML/archive.py

@@ -3,9 +3,9 @@
 """
 
 from CIME.XML.standard_module_setup import *
+from CIME.config import Config
 from CIME.XML.archive_base import ArchiveBase
 from CIME.XML.files import Files
-from CIME.utils import expect, get_model
 from copy import deepcopy
 
 logger = logging.getLogger(__name__)
@@ -28,12 +28,14 @@ def setup(self, env_archive, components, files=None):
         components_node = env_archive.make_child(
             "components", attributes={"version": "2.0"}
         )
+
         arch_components = deepcopy(components)
-        model = get_model()
-        if "drv" not in arch_components and model != "ufs":
-            arch_components.append("drv")
-        if "dart" not in arch_components and model == "cesm":
-            arch_components.append("dart")
+
+        config = Config.instance()
+
+        for comp in config.additional_archive_components:
+            if comp not in arch_components:
+                arch_components.append(comp)
 
         for comp in arch_components:
             infile = files.get_value("ARCHIVE_SPEC_FILE", {"component": comp})

CIME/XML/test_reporter.py

@@ -7,7 +7,6 @@
 import urllib.request
 from CIME.XML.standard_module_setup import *
 from CIME.XML.generic_xml import GenericXML
-from CIME.utils import expect, get_model
 import ssl
 
 # pylint: disable=protected-access
@@ -19,11 +18,6 @@ def __init__(self):
         """
         initialize an object
         """
-
-        expect(
-            get_model() == "cesm",
-            "testreport is only meant to populate the CESM test database.",
-        )
         self.root = None
 
         GenericXML.__init__(

CIME/bless_test_results.py

@@ -3,10 +3,10 @@
 from CIME.utils import (
     run_cmd,
     get_scripts_root,
-    get_model,
     EnvironmentContext,
     parse_test_name,
 )
+from CIME.config import Config
 from CIME.test_status import *
 from CIME.hist_utils import generate_baseline, compare_baseline
 from CIME.case import Case
@@ -36,10 +36,11 @@ def bless_namelists(
     if not report_only and (
         force or input("Update namelists (y/n)? ").upper() in ["Y", "YES"]
     ):
+        config = Config.instance()
 
         create_test_gen_args = " -g {} ".format(
             baseline_name
-            if get_model() == "cesm"
+            if config.create_test_flag_mode == "cesm"
             else " -g -b {} ".format(baseline_name)
         )
         if new_test_root is not None:

CIME/build.py

@@ -19,12 +19,14 @@
     get_logging_options,
     import_from_file,
 )
-from CIME.provenance import save_build_provenance as save_build_provenance_sub
+from CIME.config import Config
 from CIME.locked_files import lock_file, unlock_file
 from CIME.XML.files import Files
 
 logger = logging.getLogger(__name__)
 
+config = Config.instance()
+
 _CMD_ARGS_FOR_BUILD = (
     "CASEROOT",
     "CASETOOLS",
@@ -283,7 +285,7 @@ def uses_kokkos(case):
     cam_target = case.get_value("CAM_TARGET")
     # atm_comp   = case.get_value("COMP_ATM") # scream does not use the shared kokkoslib for now
 
-    return get_model() == "e3sm" and cam_target in (
+    return config.use_kokkos and cam_target in (
         "preqx_kokkos",
         "theta-l",
         "theta-l_kokkos",
@@ -323,7 +325,7 @@ def _build_model(
         # special case for clm
         # clm 4_5 and newer is a shared (as in sharedlibs, shared by all tests) library
         # (but not in E3SM) and should be built in build_libraries
-        if get_model() != "e3sm" and comp == "clm":
+        if config.shared_clm_component and comp == "clm":
             continue
         else:
             logger.info("         - Building {} Library ".format(model))
@@ -379,7 +381,7 @@ def _build_model(
         file_build = os.path.join(exeroot, "{}.bldlog.{}".format(cime_model, lid))
 
         ufs_driver = os.environ.get("UFS_DRIVER")
-        if cime_model == "ufs" and ufs_driver == "nems":
+        if config.ufs_alternative_config and ufs_driver == "nems":
             config_dir = os.path.join(
                 cimeroot, os.pardir, "src", "model", "NEMS", "cime", "cime_config"
             )
@@ -813,7 +815,7 @@ def _build_libraries(
                     logger.warning(line)
 
     # clm not a shared lib for E3SM
-    if get_model() != "e3sm" and (buildlist is None or "lnd" in buildlist):
+    if config.shared_clm_component and (buildlist is None or "lnd" in buildlist):
         comp_lnd = case.get_value("COMP_LND")
         if comp_lnd == "clm":
             logging.info("         - Building clm library ")
@@ -888,7 +890,7 @@ def _build_model_thread(
         libroot=libroot,
         bldroot=bldroot,
     )
-    if get_model() != "ufs":
+    if config.enable_smp:
         compile_cmd = "SMP={} {}".format(stringify_bool(smp), compile_cmd)
 
     if is_python_executable(cmd):
@@ -1104,6 +1106,7 @@ def _case_build_impl(
     os.environ["BUILD_THREADED"] = stringify_bool(build_threaded)
     cime_model = get_model()
 
+    # TODO need some other method than a flag.
     if cime_model == "e3sm" and mach == "titan" and compiler == "pgiacc":
         case.set_value("CAM_TARGET", "preqx_acc")
 
@@ -1175,7 +1178,7 @@ def _case_build_impl(
         )
 
     if not sharedlib_only:
-        if get_model() == "e3sm":
+        if config.build_model_use_cmake:
             logs.extend(
                 _build_model_cmake(
                     exeroot,
@@ -1242,7 +1245,10 @@ def post_build(case, logs, build_complete=False, save_build_provenance=True):
             os.environ["LID"] if "LID" in os.environ else get_timestamp("%y%m%d-%H%M%S")
         )
         if save_build_provenance:
-            save_build_provenance_sub(case, lid=lid)
+            try:
+                Config.instance().save_build_provenance(case, lid=lid)
+            except AttributeError:
+                logger.debug("No handler for save_build_provenance was found")
         # Set XML to indicate build complete
         case.set_value("BUILD_COMPLETE", True)
         case.set_value("BUILD_STATUS", 0)