diff --git a/test/easyconfigs/easyconfigs.py b/test/easyconfigs/easyconfigs.py
new file mode 100644
index 000000000000..0fb05afe8762
--- /dev/null
+++ b/test/easyconfigs/easyconfigs.py
@@ -0,0 +1,1510 @@
+##
+# Copyright 2013-2022 Ghent University
+#
+# This file is part of EasyBuild,
+# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
+# with support of Ghent University (http://ugent.be/hpc),
+# the Flemish Supercomputer Centre (VSC) (https://www.vscentrum.be),
+# Flemish Research Foundation (FWO) (http://www.fwo.be/en)
+# and the Department of Economy, Science and Innovation (EWI) (http://www.ewi-vlaanderen.be/en).
+#
+# https://github.com/easybuilders/easybuild
+#
+# EasyBuild is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation v2.
+#
+# EasyBuild is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with EasyBuild. If not, see .
+##
+"""
+Unit tests for easyconfig files.
+
+@author: Kenneth Hoste (Ghent University)
+"""
+import glob
+import os
+import re
+import shutil
+import sys
+import tempfile
+from distutils.version import LooseVersion
+from unittest import TestCase, TestLoader, main, skip
+
+import easybuild.main as eb_main
+import easybuild.tools.options as eboptions
+from easybuild.base import fancylogger
+from easybuild.easyblocks.generic.configuremake import ConfigureMake
+from easybuild.easyblocks.generic.pythonpackage import PythonPackage
+from easybuild.framework.easyblock import EasyBlock
+from easybuild.framework.easyconfig.constants import EASYCONFIG_CONSTANTS
+from easybuild.framework.easyconfig.default import DEFAULT_CONFIG
+from easybuild.framework.easyconfig.format.format import DEPENDENCY_PARAMETERS
+from easybuild.framework.easyconfig.easyconfig import get_easyblock_class, letter_dir_for
+from easybuild.framework.easyconfig.easyconfig import resolve_template
+from easybuild.framework.easyconfig.parser import EasyConfigParser, fetch_parameters_from_easyconfig
+from easybuild.framework.easyconfig.tools import check_sha256_checksums, dep_graph, get_paths_for, process_easyconfig
+from easybuild.tools import config
+from easybuild.tools.build_log import EasyBuildError
+from easybuild.tools.config import GENERAL_CLASS, build_option
+from easybuild.tools.filetools import change_dir, is_generic_easyblock, read_file, remove_file
+from easybuild.tools.filetools import verify_checksum, which, write_file
+from easybuild.tools.module_naming_scheme.utilities import det_full_ec_version
+from easybuild.tools.modules import modules_tool
+from easybuild.tools.py2vs3 import string_type, urlopen
+from easybuild.tools.robot import check_conflicts, resolve_dependencies
+from easybuild.tools.run import run_cmd
+from easybuild.tools.options import set_tmpdir
+from easybuild.tools.utilities import nub
+
+
+# indicates whether all the single tests are OK,
+# and that bigger tests (building dep graph, testing for conflicts, ...) can be run as well
+# other than optimizing for time, this also helps to get around problems like http://bugs.python.org/issue10949
+single_tests_ok = True
+
+
+def is_pr():
+ """Return true if run in a pull request CI"""
+ # $TRAVIS_PULL_REQUEST should be a PR number, otherwise we're not running tests for a PR
+ travis_pr_test = re.match('^[0-9]+$', os.environ.get('TRAVIS_PULL_REQUEST', ''))
+
+ # when testing a PR in GitHub Actions, $GITHUB_EVENT_NAME will be set to 'pull_request'
+ github_pr_test = os.environ.get('GITHUB_EVENT_NAME') == 'pull_request'
+ return travis_pr_test or github_pr_test
+
+
+def get_target_branch():
+ """Return the target branch of a pull request"""
+ # target branch should be anything other than 'master';
+ # usually is 'develop', but could also be a release branch like '3.7.x'
+ target_branch = os.environ.get('GITHUB_BASE_REF', None)
+ if not target_branch:
+ target_branch = os.environ.get('TRAVIS_BRANCH', None)
+ if not target_branch:
+ raise RuntimeError("Did not find a target branch")
+ return target_branch
+
+
+def skip_if_not_pr_to_non_main_branch():
+ if not is_pr():
+ return skip("Only run for pull requests")
+ if get_target_branch() == "main":
+ return skip("Not run for pull requests against main")
+ return lambda func: func
+
+
+def get_files_from_diff(diff_filter, ext, basename=True):
+ """Return the files changed on HEAD relative to the current target branch"""
+ target_branch = get_target_branch()
+
+ # relocate to top-level directory of repository to run 'git diff' command
+ top_dir = os.path.dirname(os.path.dirname(get_paths_for('easyconfigs')[0]))
+ cwd = change_dir(top_dir)
+
+ # first determine the 'merge base' between target branch and PR branch
+ # cfr. https://git-scm.com/docs/git-merge-base
+ cmd = "git merge-base %s HEAD" % target_branch
+ out, ec = run_cmd(cmd, simple=False, log_ok=False)
+ if ec == 0:
+ merge_base = out.strip()
+ print("Merge base for %s and HEAD: %s" % (target_branch, merge_base))
+ else:
+ msg = "Failed to determine merge base (ec: %s, output: '%s'), "
+ msg += "falling back to specifying target branch %s"
+ print(msg % (ec, out, target_branch))
+ merge_base = target_branch
+
+ # determine list of changed files using 'git diff' and merge base determined above
+ cmd = "git diff --name-only --diff-filter=%s %s..HEAD --" % (diff_filter, merge_base)
+ out, _ = run_cmd(cmd, simple=False)
+ if basename:
+ files = [os.path.basename(f) for f in out.strip().split('\n') if f.endswith(ext)]
+ else:
+ files = [f for f in out.strip().split('\n') if f.endswith(ext)]
+
+ change_dir(cwd)
+ return files
+
+
+def get_eb_files_from_diff(diff_filter):
+ """Return the easyconfig files changed on HEAD relative to the current target branch"""
+ return get_files_from_diff(diff_filter, '.eb')
+
+
+class EasyConfigTest(TestCase):
+ """Baseclass for easyconfig testcases."""
+
+ @classmethod
+ def setUpClass(cls):
+ """Setup environment for all tests. Called once!"""
+ # make sure that the EasyBuild installation is still known even if we purge an EB module
+ if os.getenv('EB_SCRIPT_PATH') is None:
+ eb_path = which('eb')
+ if eb_path is not None:
+ os.environ['EB_SCRIPT_PATH'] = eb_path
+
+ # initialize configuration (required for e.g. default modules_tool setting)
+ eb_go = eboptions.parse_options(args=[]) # Ignore cmdline args as those are meant for the unittest framework
+ config.init(eb_go.options, eb_go.get_options_by_section('config'))
+ build_options = {
+ 'check_osdeps': False,
+ 'external_modules_metadata': {},
+ 'force': True,
+ 'local_var_naming_check': 'error',
+ 'optarch': 'test',
+ 'robot_path': get_paths_for("easyconfigs")[0],
+ 'silent': True,
+ 'suffix_modules_path': GENERAL_CLASS,
+ 'valid_module_classes': config.module_classes(),
+ 'valid_stops': [x[0] for x in EasyBlock.get_steps()],
+ }
+ config.init_build_options(build_options=build_options)
+ set_tmpdir()
+
+ # put dummy 'craype-test' module in place, which is required for parsing easyconfigs using Cray* toolchains
+ cls.TMPDIR = tempfile.mkdtemp()
+ os.environ['MODULEPATH'] = cls.TMPDIR
+ write_file(os.path.join(cls.TMPDIR, 'craype-test'), '#%Module\n')
+
+ log = fancylogger.getLogger("EasyConfigTest", fname=False)
+
+ # make sure a logger is present for main
+ eb_main._log = log
+
+ cls._ordered_specs = None
+ cls._parsed_easyconfigs = []
+ cls._parsed_all_easyconfigs = False
+ cls._changed_ecs = None # easyconfigs changed in a PR
+ cls._changed_patches = None # patches changed in a PR
+
+ @classmethod
+ def tearDownClass(cls):
+ """Cleanup after running all tests"""
+ shutil.rmtree(cls.TMPDIR)
+
+ @classmethod
+ def parse_all_easyconfigs(cls):
+ """Parse all easyconfigs."""
+ if cls._parsed_all_easyconfigs:
+ return
+ # all available easyconfig files
+ easyconfigs_path = get_paths_for("easyconfigs")[0]
+ specs = glob.glob('%s/*/*/*.eb' % easyconfigs_path)
+ parsed_specs = set(ec['spec'] for ec in cls._parsed_easyconfigs)
+ for spec in specs:
+ if spec not in parsed_specs:
+ cls._parsed_easyconfigs.extend(process_easyconfig(spec))
+ cls._parsed_all_easyconfigs = True
+
+ @classmethod
+ def resolve_all_dependencies(cls):
+ """Resolve dependencies between easyconfigs"""
+ # Parse all easyconfigs if not done yet
+ cls.parse_all_easyconfigs()
+ # filter out external modules
+ for ec in cls._parsed_easyconfigs:
+ for dep in ec['dependencies'][:]:
+ if dep.get('external_module', False):
+ ec['dependencies'].remove(dep)
+ cls._ordered_specs = resolve_dependencies(
+ cls._parsed_easyconfigs, modules_tool(), retain_all_deps=True)
+
+ def _get_changed_easyconfigs(self):
+ """Gather all added or modified easyconfigs"""
+ # get list of changed easyconfigs
+ changed_ecs_filenames = get_eb_files_from_diff(diff_filter='M')
+ added_ecs_filenames = get_eb_files_from_diff(diff_filter='A')
+ if changed_ecs_filenames:
+ print("\nList of changed easyconfig files in this PR:\n\t%s" % '\n\t'.join(changed_ecs_filenames))
+ if added_ecs_filenames:
+ print("\nList of added easyconfig files in this PR:\n\t%s" % '\n\t'.join(added_ecs_filenames))
+ EasyConfigTest._changed_ecs_filenames = changed_ecs_filenames
+ EasyConfigTest._added_ecs_filenames = added_ecs_filenames
+
+ # grab parsed easyconfigs for changed easyconfig files
+ changed_ecs = []
+ for ec_fn in changed_ecs_filenames + added_ecs_filenames:
+ match = None
+ for ec in self.parsed_easyconfigs:
+ if os.path.basename(ec['spec']) == ec_fn:
+ match = ec['ec']
+ break
+
+ if match:
+ changed_ecs.append(match)
+ else:
+ # if no easyconfig is found, it's possible some archived easyconfigs were touched in the PR...
+ # so as a last resort, try to find the easyconfig file in __archive__
+ easyconfigs_path = get_paths_for("easyconfigs")[0]
+ specs = glob.glob('%s/__archive__/*/*/%s' % (easyconfigs_path, ec_fn))
+ if len(specs) == 1:
+ ec = process_easyconfig(specs[0])[0]
+ changed_ecs.append(ec['ec'])
+ else:
+ raise RuntimeError("Failed to find parsed easyconfig for %s"
+ " (and could not isolate it in easyconfigs archive either)" % ec_fn)
+ EasyConfigTest._changed_ecs = changed_ecs
+
+ def _get_changed_patches(self):
+ """Gather all added or modified patches"""
+
+ # get list of changed/added patch files
+ changed_patches_filenames = get_files_from_diff(diff_filter='M', ext='.patch', basename=False)
+ added_patches_filenames = get_files_from_diff(diff_filter='A', ext='.patch', basename=False)
+
+ if changed_patches_filenames:
+ print("\nList of changed patch files in this PR:\n\t%s" % '\n\t'.join(changed_patches_filenames))
+ if added_patches_filenames:
+ print("\nList of added patch files in this PR:\n\t%s" % '\n\t'.join(added_patches_filenames))
+
+ EasyConfigTest._changed_patches = changed_patches_filenames + added_patches_filenames
+
+ @property
+ def parsed_easyconfigs(self):
+ # parse all easyconfigs if they haven't been already
+ EasyConfigTest.parse_all_easyconfigs()
+ return EasyConfigTest._parsed_easyconfigs
+
+ @property
+ def ordered_specs(self):
+ # Resolve dependencies if not done
+ if EasyConfigTest._ordered_specs is None:
+ EasyConfigTest.resolve_all_dependencies()
+ return EasyConfigTest._ordered_specs
+
+ @property
+ def changed_ecs_filenames(self):
+ if EasyConfigTest._changed_ecs is None:
+ self._get_changed_easyconfigs()
+ return EasyConfigTest._changed_ecs_filenames
+
+ @property
+ def added_ecs_filenames(self):
+ if EasyConfigTest._changed_ecs is None:
+ self._get_changed_easyconfigs()
+ return EasyConfigTest._added_ecs_filenames
+
+ @property
+ def changed_ecs(self):
+ if EasyConfigTest._changed_ecs is None:
+ self._get_changed_easyconfigs()
+ return EasyConfigTest._changed_ecs
+
+ @property
+ def changed_patches(self):
+ if EasyConfigTest._changed_patches is None:
+ self._get_changed_patches()
+ return EasyConfigTest._changed_patches
+
+ def test_dep_graph(self):
+ """Unit test that builds a full dependency graph."""
+ # pygraph dependencies required for constructing dependency graph are not available prior to Python 2.6
+ if LooseVersion(sys.version) >= LooseVersion('2.6') and single_tests_ok:
+ # temporary file for dep graph
+ (hn, fn) = tempfile.mkstemp(suffix='.dot')
+ os.close(hn)
+
+ dep_graph(fn, self.ordered_specs)
+
+ remove_file(fn)
+ else:
+ print("(skipped dep graph test)")
+
+ def test_conflicts(self):
+ """Check whether any conflicts occur in software dependency graphs."""
+
+ if not single_tests_ok:
+ print("(skipped conflicts test)")
+ return
+
+ self.assertFalse(check_conflicts(self.ordered_specs, modules_tool(), check_inter_ec_conflicts=False),
+ "No conflicts detected")
+
+ def test_deps(self):
+ """Perform checks on dependencies in easyconfig files"""
+
+ fails = []
+
+ for ec in self.parsed_easyconfigs:
+ # make sure that no odd versions (like 1.13) of HDF5 are used as a dependency,
+ # since those are released candidates - only even versions (like 1.12) are stable releases;
+ # see https://docs.hdfgroup.org/archive/support/HDF5/doc/TechNotes/Version.html
+ for dep in ec['ec'].dependencies():
+ if dep['name'] == 'HDF5':
+ ver = dep['version']
+ if int(ver.split('.')[1]) % 2 == 1:
+ fail = "Odd minor versions of HDF5 should not be used as a dependency: "
+ fail += "found HDF5 v%s as dependency in %s" % (ver, os.path.basename(ec['spec']))
+ fails.append(fail)
+
+ self.assertFalse(len(fails), '\n'.join(sorted(fails)))
+
+ def check_dep_vars(self, gen, dep, dep_vars):
+ """Check whether available variants of a particular dependency are acceptable or not."""
+
+ # 'guilty' until proven 'innocent'
+ res = False
+
+ # filter out wrapped Java versions
+ # i.e. if the version of one is a prefix of the version of the other one (e.g. 1.8 & 1.8.0_181)
+ if dep == 'Java':
+ dep_vars_to_check = sorted(dep_vars.keys())
+
+ retained_dep_vars = []
+
+ while dep_vars_to_check:
+ dep_var = dep_vars_to_check.pop()
+ dep_var_version = dep_var.split(';')[0]
+
+ # remove dep vars wrapped by current dep var
+ dep_vars_to_check = [x for x in dep_vars_to_check if not x.startswith(dep_var_version + '.')]
+
+ retained_dep_vars = [x for x in retained_dep_vars if not x.startswith(dep_var_version + '.')]
+
+ retained_dep_vars.append(dep_var)
+
+ for key in list(dep_vars.keys()):
+ if key not in retained_dep_vars:
+ del dep_vars[key]
+
+ version_regex = re.compile('^version: (?P[^;]+);')
+
+ # filter out binutils with empty versionsuffix which is used to build toolchain compiler
+ if dep == 'binutils' and len(dep_vars) > 1:
+ empty_vsuff_vars = [v for v in dep_vars.keys() if v.endswith('versionsuffix: ')]
+ if len(empty_vsuff_vars) == 1:
+ dep_vars = dict((k, v) for (k, v) in dep_vars.items() if k != empty_vsuff_vars[0])
+
+ # multiple variants of HTSlib is OK as long as they are deps for a matching version of BCFtools;
+ # same goes for WRF and WPS; Gurobi and Rgurobi
+ for dep_name, parent_name in [('HTSlib', 'BCFtools'), ('WRF', 'WPS'), ('Gurobi', 'Rgurobi')]:
+ if dep == dep_name and len(dep_vars) > 1:
+ for key in list(dep_vars):
+ ecs = dep_vars[key]
+ # filter out dep variants that are only used as dependency for parent with same version
+ dep_ver = version_regex.search(key).group('version')
+ if all(ec.startswith('%s-%s-' % (parent_name, dep_ver)) for ec in ecs) and len(dep_vars) > 1:
+ dep_vars.pop(key)
+
+ # multiple versions of Boost is OK as long as they are deps for a matching Boost.Python
+ if dep == 'Boost' and len(dep_vars) > 1:
+ for key in list(dep_vars):
+ ecs = dep_vars[key]
+ # filter out Boost variants that are only used as dependency for Boost.Python with same version
+ boost_ver = version_regex.search(key).group('version')
+ if all(ec.startswith('Boost.Python-%s-' % boost_ver) for ec in ecs):
+ dep_vars.pop(key)
+
+ # filter out Perl with -minimal versionsuffix which are only used in makeinfo-minimal
+ if dep == 'Perl':
+ minimal_vsuff_vars = [v for v in dep_vars.keys() if v.endswith('versionsuffix: -minimal')]
+ if len(minimal_vsuff_vars) == 1:
+ dep_vars = dict((k, v) for (k, v) in dep_vars.items() if k != minimal_vsuff_vars[0])
+
+ # filter out FFTW and imkl with -serial versionsuffix which are used in non-MPI subtoolchains
+ if dep in ['FFTW', 'imkl']:
+ serial_vsuff_vars = [v for v in dep_vars.keys() if v.endswith('versionsuffix: -serial')]
+ if len(serial_vsuff_vars) == 1:
+ dep_vars = dict((k, v) for (k, v) in dep_vars.items() if k != serial_vsuff_vars[0])
+
+ # filter out BLIS and libFLAME with -amd versionsuffix
+ # (AMD forks, used in gobff/*-amd toolchains)
+ if dep in ['BLIS', 'libFLAME']:
+ amd_vsuff_vars = [v for v in dep_vars.keys() if v.endswith('versionsuffix: -amd')]
+ if len(amd_vsuff_vars) == 1:
+ dep_vars = dict((k, v) for (k, v) in dep_vars.items() if k != amd_vsuff_vars[0])
+
+ # filter out ScaLAPACK with -BLIS-* versionsuffix, used in goblf toolchain
+ if dep == 'ScaLAPACK':
+ blis_vsuff_vars = [v for v in dep_vars.keys() if '; versionsuffix: -BLIS-' in v]
+ if len(blis_vsuff_vars) == 1:
+ dep_vars = dict((k, v) for (k, v) in dep_vars.items() if k != blis_vsuff_vars[0])
+
+ if dep == 'ScaLAPACK':
+ # filter out ScaLAPACK with -bf versionsuffix, used in gobff toolchain
+ bf_vsuff_vars = [v for v in dep_vars.keys() if '; versionsuffix: -bf' in v]
+ if len(bf_vsuff_vars) == 1:
+ dep_vars = dict((k, v) for (k, v) in dep_vars.items() if k != bf_vsuff_vars[0])
+ # filter out ScaLAPACK with -bl versionsuffix, used in goblf toolchain
+ bl_vsuff_vars = [v for v in dep_vars.keys() if '; versionsuffix: -bl' in v]
+ if len(bl_vsuff_vars) == 1:
+ dep_vars = dict((k, v) for (k, v) in dep_vars.items() if k != bl_vsuff_vars[0])
+
+ # filter out HDF5 with -serial versionsuffix which is used in HDF5 for Python (h5py)
+ if dep in ['HDF5']:
+ serial_vsuff_vars = [v for v in dep_vars.keys() if v.endswith('versionsuffix: -serial')]
+ if len(serial_vsuff_vars) == 1:
+ dep_vars = dict((k, v) for (k, v) in dep_vars.items() if k != serial_vsuff_vars[0])
+
+ # for some dependencies, we allow exceptions for software that depends on a particular version,
+ # as long as that's indicated by the versionsuffix
+ versionsuffix_deps = ['ASE', 'Boost', 'CUDAcore', 'Java', 'Lua',
+ 'PLUMED', 'PyTorch', 'R', 'TensorFlow']
+ if dep in versionsuffix_deps and len(dep_vars) > 1:
+
+ # check for '-CUDA-*' versionsuffix for CUDAcore dependency
+ if dep == 'CUDAcore':
+ dep = 'CUDA'
+
+ for key in list(dep_vars):
+ dep_ver = version_regex.search(key).group('version')
+ # use version of Java wrapper rather than full Java version
+ if dep == 'Java':
+ dep_ver = '.'.join(dep_ver.split('.')[:2])
+ # filter out dep version if all easyconfig filenames using it include specific dep version
+ if all(re.search('-%s-%s' % (dep, dep_ver), v) for v in dep_vars[key]):
+ dep_vars.pop(key)
+ # always retain at least one dep variant
+ if len(dep_vars) == 1:
+ break
+
+ # filter R dep for a specific version of Python 2.x
+ if dep == 'R' and len(dep_vars) > 1:
+ for key in list(dep_vars):
+ if '; versionsuffix: -Python-2' in key:
+ dep_vars.pop(key)
+ # always retain at least one variant
+ if len(dep_vars) == 1:
+ break
+
+ # filter out variants that are specific to a particular version of CUDA
+ cuda_dep_vars = [v for v in dep_vars.keys() if '-CUDA' in v]
+ if len(dep_vars) >= len(cuda_dep_vars) and len(dep_vars) > 1:
+ for key in list(dep_vars):
+ if re.search('; versionsuffix: .*-CUDA-[0-9.]+', key):
+ dep_vars.pop(key)
+ # always retain at least one dep variant
+ if len(dep_vars) == 1:
+ break
+
+ # some software packages require a specific (older/newer) version of a particular dependency
+ alt_dep_versions = {
+ 'jax': [(r'0\.3\.9', [r'AlphaFold-2\.2\.2-'])],
+ # arrow-R 6.0.0.2 is used for two R/R-bundle-Bioconductor sets (4.1.2/3.14 and 4.2.0/3.15)
+ 'arrow-R': [('6.0.0.2', [r'R-bundle-Bioconductor-'])],
+ # EMAN2 2.3 requires Boost(.Python) 1.64.0
+ 'Boost': [('1.64.0;', [r'Boost.Python-1\.64\.0-', r'EMAN2-2\.3-'])],
+ 'Boost.Python': [('1.64.0;', [r'EMAN2-2\.3-'])],
+ # GATE 9.2 requires CHLEP 2.4.5.1 and Geant4 11.0.x
+ 'CLHEP': [('2.4.5.1;', [r'GATE-9\.2-foss-2021b'])],
+ 'Geant4': [('11.0.1;', [r'GATE-9\.2-foss-2021b'])],
+ # ncbi-vdb v2.x require HDF5 v1.10.x (HISAT2, SKESA, shovill depend on ncbi-vdb)
+ 'HDF5': [(r'1\.10\.', [r'ncbi-vdb-2\.11\.', r'HISAT2-2\.2\.', r'SKESA-2\.4\.', r'shovill-1\.1\.'])],
+ # VMTK 1.4.x requires ITK 4.13.x
+ 'ITK': [(r'4\.13\.', [r'VMTK-1\.4\.'])],
+ # Kraken 1.x requires Jellyfish 1.x (Roary & metaWRAP depend on Kraken 1.x)
+ 'Jellyfish': [(r'1\.', [r'Kraken-1\.', r'Roary-3\.12\.0', r'metaWRAP-1\.2'])],
+ # Libint 1.1.6 is required by older CP2K versions
+ 'Libint': [(r'1\.1\.6', [r'CP2K-[3-6]'])],
+ # libxc 2.x or 3.x is required by ABINIT, AtomPAW, CP2K, GPAW, horton, PySCF, WIEN2k
+ # libxc 4.x is required by libGridXC
+ # (Qiskit depends on PySCF), Elk 7.x requires libxc >= 5
+ 'libxc': [
+ (r'[23]\.', [r'ABINIT-', r'AtomPAW-', r'CP2K-', r'GPAW-', r'horton-',
+ r'PySCF-', r'Qiskit-', r'WIEN2k-']),
+ (r'4\.', [r'libGridXC-']),
+ (r'5\.', [r'Elk-']),
+ ],
+ # some software depends on numba, which typically requires an older LLVM;
+ # this includes BirdNET, cell2location, cryoDRGN, librosa, PyOD, Python-Geometric, scVelo, scanpy
+ 'LLVM': [
+ # numba 0.47.x requires LLVM 7.x or 8.x (see https://github.com/numba/llvmlite#compatibility)
+ (r'8\.', [r'numba-0\.47\.0-', r'librosa-0\.7\.2-', r'BirdNET-20201214-',
+ r'scVelo-0\.1\.24-', r'PyTorch-Geometric-1\.[346]\.[23]']),
+ (r'10\.0\.1', [r'cell2location-0\.05-alpha-', r'cryoDRGN-0\.3\.2-', r'loompy-3\.0\.6-',
+ r'numba-0\.52\.0-', r'PyOD-0\.8\.7-', r'PyTorch-Geometric-1\.6\.3',
+ r'scanpy-1\.7\.2-', r'umap-learn-0\.4\.6-']),
+ ],
+ 'Lua': [
+ # SimpleITK 2.1.0 requires Lua 5.3.x, MedPy and nnU-Net depend on SimpleITK
+ (r'5\.3\.5', [r'nnU-Net-1\.7\.0-', r'MedPy-0\.4\.0-', r'SimpleITK-2\.1\.0-']),
+ ],
+ # TensorFlow 2.5+ requires a more recent NCCL than version 2.4.8 used in 2019b generation;
+ # Horovod depends on TensorFlow, so same exception required there
+ 'NCCL': [(r'2\.11\.4', [r'TensorFlow-2\.[5-9]\.', r'Horovod-0\.2[2-9]'])],
+ # rampart requires nodejs > 10, artic-ncov2019 requires rampart
+ 'nodejs': [('12.16.1', ['rampart-1.2.0rc3-', 'artic-ncov2019-2020.04.13'])],
+ # some software depends on an older numba;
+ # this includes BirdNET, cell2location, cryoDRGN, librosa, PyOD, Python-Geometric, scVelo, scanpy
+ 'numba': [
+ (r'0\.52\.0', [r'cell2location-0\.05-alpha-', r'cryoDRGN-0\.3\.2-', r'loompy-3\.0\.6-',
+ r'PyOD-0\.8\.7-', r'PyTorch-Geometric-1\.6\.3', r'scanpy-1\.7\.2-',
+ r'umap-learn-0\.4\.6-']),
+ ],
+ # medaka 1.1.*, 1.2.*, 1.4.* requires Pysam 0.16.0.1,
+ # which is newer than what others use as dependency w.r.t. Pysam version in 2019b generation;
+ # decona 0.1.2 and NGSpeciesID 0.1.1.1 depend on medaka 1.1.3
+ # WhatsHap 1.4 + medaka 1.6.0 require Pysam >= 0.18.0 (NGSpeciesID depends on medaka)
+ 'Pysam': [
+ ('0.16.0.1;', ['medaka-1.2.[0]-', 'medaka-1.1.[13]-', 'medaka-1.4.3-', 'decona-0.1.2-',
+ 'NGSpeciesID-0.1.1.1-']),
+ ('0.18.0;', ['medaka-1.6.0-', 'NGSpeciesID-0.1.2.1-', 'WhatsHap-1.4-']),
+ ],
+ # OPERA requires SAMtools 0.x
+ 'SAMtools': [(r'0\.', [r'ChimPipe-0\.9\.5', r'Cufflinks-2\.2\.1', r'OPERA-2\.0\.6',
+ r'CGmapTools-0\.1\.2', r'BatMeth2-2\.1'])],
+ # NanoPlot, NanoComp use an older version of Seaborn
+ 'Seaborn': [(r'0\.10\.1', [r'NanoComp-1\.13\.1-', r'NanoPlot-1\.33\.0-'])],
+ # Shasta requires spoa 3.x
+ 'spoa': [(r'3\.4\.0', [r'Shasta-0\.8\.0-'])],
+ # UShER requires tbb-2020.3 as newer versions will not build
+ 'tbb': [('2020.3', ['UShER-0.5.0-'])],
+ 'TensorFlow': [
+ # medaka 0.11.4/0.12.0 requires recent TensorFlow <= 1.14 (and Python 3.6),
+ # artic-ncov2019 requires medaka
+ ('1.13.1;', ['medaka-0.11.4-', 'medaka-0.12.0-', 'artic-ncov2019-2020.04.13']),
+ # medaka 1.1.* and 1.2.* requires TensorFlow 2.2.0
+ # (while other 2019b easyconfigs use TensorFlow 2.1.0 as dep);
+ # TensorFlow 2.2.0 is also used as a dep for Horovod 0.19.5;
+ # decona 0.1.2 and NGSpeciesID 0.1.1.1 depend on medaka 1.1.3
+ ('2.2.0;', ['medaka-1.2.[0]-', 'medaka-1.1.[13]-', 'Horovod-0.19.5-', 'decona-0.1.2-',
+ 'NGSpeciesID-0.1.1.1-']),
+ # medaka 1.4.3 (foss/2019b) depends on TensorFlow 2.2.2
+ ('2.2.2;', ['medaka-1.4.3-']),
+ # medaka 1.4.3 (foss/2020b) depends on TensorFlow 2.2.3; longread_umi and artic depend on medaka
+ ('2.2.3;', ['medaka-1.4.3-', 'artic-ncov2019-2021.06.24-', 'longread_umi-0.3.2-']),
+ # AlphaFold 2.1.2 (foss/2020b) depends on TensorFlow 2.5.0
+ ('2.5.0;', ['AlphaFold-2.1.2-']),
+ # medaka 1.5.0 (foss/2021a) depends on TensorFlow >=2.5.2, <2.6.0
+ ('2.5.3;', ['medaka-1.5.0-']),
+ ],
+ # smooth-topk uses a newer version of torchvision
+ 'torchvision': [('0.11.3;', ['smooth-topk-1.0-20210817-'])],
+ # for the sake of backwards compatibility, keep UCX-CUDA v1.11.0 which depends on UCX v1.11.0
+ # (for 2021b, UCX was updated to v1.11.2)
+ 'UCX': [('1.11.0;', ['UCX-CUDA-1.11.0-'])],
+ # WPS 3.9.1 requires WRF 3.9.1.1
+ 'WRF': [(r'3\.9\.1\.1', [r'WPS-3\.9\.1'])],
+ }
+ if dep in alt_dep_versions and len(dep_vars) > 1:
+ for key in list(dep_vars):
+ for version_pattern, parents in alt_dep_versions[dep]:
+ # filter out known alternative dependency versions
+ if re.search('^version: %s' % version_pattern, key):
+ # only filter if the easyconfig using this dep variants is known
+ if all(any(re.search(p, x) for p in parents) for x in dep_vars[key]):
+ dep_vars.pop(key)
+
+ # filter out ELSI variants with -PEXSI suffix
+ if dep == 'ELSI' and len(dep_vars) > 1:
+ pexsi_vsuff_vars = [v for v in dep_vars.keys() if v.endswith('versionsuffix: -PEXSI')]
+ if len(pexsi_vsuff_vars) == 1:
+ dep_vars = dict((k, v) for (k, v) in dep_vars.items() if k != pexsi_vsuff_vars[0])
+
+ # only single variant is always OK
+ if len(dep_vars) == 1:
+ res = True
+
+ elif len(dep_vars) == 2 and dep in ['Python', 'Tkinter']:
+ # for Python & Tkinter, it's OK to have on 2.x and one 3.x version
+ v2_dep_vars = [x for x in dep_vars.keys() if x.startswith('version: 2.')]
+ v3_dep_vars = [x for x in dep_vars.keys() if x.startswith('version: 3.')]
+ if len(v2_dep_vars) == 1 and len(v3_dep_vars) == 1:
+ res = True
+
+ # two variants is OK if one is for Python 2.x and the other is for Python 3.x (based on versionsuffix)
+ elif len(dep_vars) == 2:
+ py2_dep_vars = [x for x in dep_vars.keys() if '; versionsuffix: -Python-2.' in x]
+ py3_dep_vars = [x for x in dep_vars.keys() if '; versionsuffix: -Python-3.' in x]
+ if len(py2_dep_vars) == 1 and len(py3_dep_vars) == 1:
+ res = True
+
+ # for recent generations, there's no versionsuffix anymore for Python 3,
+ # but we still allow variants depending on Python 2.x + 3.x
+ is_recent_gen = False
+ full_toolchain_regex = re.compile(r'^20[1-9][0-9][ab]$')
+ gcc_toolchain_regex = re.compile(r'^GCC(core)?-[0-9]?[0-9]\.[0-9]$')
+ if full_toolchain_regex.match(gen):
+ is_recent_gen = LooseVersion(gen) >= LooseVersion('2020b')
+ elif gcc_toolchain_regex.match(gen):
+ genver = gen.split('-', 1)[1]
+ is_recent_gen = LooseVersion(genver) >= LooseVersion('10.2')
+ else:
+ raise EasyBuildError("Unkown type of toolchain generation: %s" % gen)
+
+ if is_recent_gen:
+ py2_dep_vars = [x for x in dep_vars.keys() if '; versionsuffix: -Python-2.' in x]
+ py3_dep_vars = [x for x in dep_vars.keys() if x.strip().endswith('; versionsuffix:')]
+ if len(py2_dep_vars) == 1 and len(py3_dep_vars) == 1:
+ res = True
+
+ return res
+
+ def test_check_dep_vars(self):
+ """Test check_dep_vars utility method."""
+
+ # one single dep version: OK
+ self.assertTrue(self.check_dep_vars('2019b', 'testdep', {
+ 'version: 1.2.3; versionsuffix:': ['foo-1.2.3.eb', 'bar-4.5.6.eb'],
+ }))
+ self.assertTrue(self.check_dep_vars('2019b', 'testdep', {
+ 'version: 1.2.3; versionsuffix: -test': ['foo-1.2.3.eb', 'bar-4.5.6.eb'],
+ }))
+
+ # two or more dep versions (no special case: not OK)
+ self.assertFalse(self.check_dep_vars('2019b', 'testdep', {
+ 'version: 1.2.3; versionsuffix:': ['foo-1.2.3.eb'],
+ 'version: 4.5.6; versionsuffix:': ['bar-4.5.6.eb'],
+ }))
+ self.assertFalse(self.check_dep_vars('2019b', 'testdep', {
+ 'version: 0.0; versionsuffix:': ['foobar-0.0.eb'],
+ 'version: 1.2.3; versionsuffix:': ['foo-1.2.3.eb'],
+ 'version: 4.5.6; versionsuffix:': ['bar-4.5.6.eb'],
+ }))
+
+ # Java is a special case, with wrapped Java versions
+ self.assertTrue(self.check_dep_vars('2019b', 'Java', {
+ 'version: 1.8.0_221; versionsuffix:': ['foo-1.2.3.eb'],
+ 'version: 1.8; versionsuffix:': ['foo-1.2.3.eb'],
+ }))
+ # two Java wrappers is not OK
+ self.assertFalse(self.check_dep_vars('2019b', 'Java', {
+ 'version: 1.8.0_221; versionsuffix:': ['foo-1.2.3.eb'],
+ 'version: 1.8; versionsuffix:': ['foo-1.2.3.eb'],
+ 'version: 11.0.2; versionsuffix:': ['bar-4.5.6.eb'],
+ 'version: 11; versionsuffix:': ['bar-4.5.6.eb'],
+ }))
+ # OK to have two or more wrappers if versionsuffix is used to indicate exception
+ self.assertTrue(self.check_dep_vars('2019b', 'Java', {
+ 'version: 1.8.0_221; versionsuffix:': ['foo-1.2.3.eb'],
+ 'version: 1.8; versionsuffix:': ['foo-1.2.3.eb'],
+ 'version: 11.0.2; versionsuffix:': ['bar-4.5.6-Java-11.eb'],
+ 'version: 11; versionsuffix:': ['bar-4.5.6-Java-11.eb'],
+ }))
+ # versionsuffix must be there for all easyconfigs to indicate exception
+ self.assertFalse(self.check_dep_vars('2019b', 'Java', {
+ 'version: 1.8.0_221; versionsuffix:': ['foo-1.2.3.eb'],
+ 'version: 1.8; versionsuffix:': ['foo-1.2.3.eb'],
+ 'version: 11.0.2; versionsuffix:': ['bar-4.5.6-Java-11.eb', 'bar-4.5.6.eb'],
+ 'version: 11; versionsuffix:': ['bar-4.5.6-Java-11.eb', 'bar-4.5.6.eb'],
+ }))
+ self.assertTrue(self.check_dep_vars('2019b', 'Java', {
+ 'version: 1.8.0_221; versionsuffix:': ['foo-1.2.3.eb'],
+ 'version: 1.8; versionsuffix:': ['foo-1.2.3.eb'],
+ 'version: 11.0.2; versionsuffix:': ['bar-4.5.6-Java-11.eb'],
+ 'version: 11; versionsuffix:': ['bar-4.5.6-Java-11.eb'],
+ 'version: 12.1.6; versionsuffix:': ['foobar-0.0-Java-12.eb'],
+ 'version: 12; versionsuffix:': ['foobar-0.0-Java-12.eb'],
+ }))
+
+ # strange situation: odd number of Java versions
+ # not OK: two Java wrappers (and no versionsuffix to indicate exception)
+ self.assertFalse(self.check_dep_vars('2019b', 'Java', {
+ 'version: 1.8.0_221; versionsuffix:': ['foo-1.2.3.eb'],
+ 'version: 1.8; versionsuffix:': ['foo-1.2.3.eb'],
+ 'version: 11; versionsuffix:': ['bar-4.5.6.eb'],
+ }))
+ # OK because of -Java-11 versionsuffix
+ self.assertTrue(self.check_dep_vars('2019b', 'Java', {
+ 'version: 1.8.0_221; versionsuffix:': ['foo-1.2.3.eb'],
+ 'version: 1.8; versionsuffix:': ['foo-1.2.3.eb'],
+ 'version: 11; versionsuffix:': ['bar-4.5.6-Java-11.eb'],
+ }))
+ # not OK: two Java wrappers (and no versionsuffix to indicate exception)
+ self.assertFalse(self.check_dep_vars('2019b', 'Java', {
+ 'version: 1.8; versionsuffix:': ['foo-1.2.3.eb'],
+ 'version: 11.0.2; versionsuffix:': ['bar-4.5.6.eb'],
+ 'version: 11; versionsuffix:': ['bar-4.5.6.eb'],
+ }))
+ # OK because of -Java-11 versionsuffix
+ self.assertTrue(self.check_dep_vars('2019b', 'Java', {
+ 'version: 1.8; versionsuffix:': ['foo-1.2.3.eb'],
+ 'version: 11.0.2; versionsuffix:': ['bar-4.5.6-Java-11.eb'],
+ 'version: 11; versionsuffix:': ['bar-4.5.6-Java-11.eb'],
+ }))
+
+ # two different versions of Boost is not OK
+ self.assertFalse(self.check_dep_vars('2019b', 'Boost', {
+ 'version: 1.64.0; versionsuffix:': ['foo-1.2.3.eb'],
+ 'version: 1.70.0; versionsuffix:': ['foo-2.3.4.eb'],
+ }))
+
+ # a different Boost version that is only used as dependency for a matching Boost.Python is fine
+ self.assertTrue(self.check_dep_vars('2019a', 'Boost', {
+ 'version: 1.64.0; versionsuffix:': ['Boost.Python-1.64.0-gompi-2019a.eb'],
+ 'version: 1.70.0; versionsuffix:': ['foo-2.3.4.eb'],
+ }))
+ self.assertTrue(self.check_dep_vars('2019a', 'Boost', {
+ 'version: 1.64.0; versionsuffix:': ['Boost.Python-1.64.0-gompi-2019a.eb'],
+ 'version: 1.66.0; versionsuffix:': ['Boost.Python-1.66.0-gompi-2019a.eb'],
+ 'version: 1.70.0; versionsuffix:': ['foo-2.3.4.eb'],
+ }))
+ self.assertFalse(self.check_dep_vars('2019a', 'Boost', {
+ 'version: 1.64.0; versionsuffix:': ['Boost.Python-1.64.0-gompi-2019a.eb'],
+ 'version: 1.66.0; versionsuffix:': ['foo-1.2.3.eb'],
+ 'version: 1.70.0; versionsuffix:': ['foo-2.3.4.eb'],
+ }))
+
+ self.assertTrue(self.check_dep_vars('2018a', 'Boost', {
+ 'version: 1.63.0; versionsuffix: -Python-2.7.14': ['EMAN2-2.21a-foss-2018a-Python-2.7.14-Boost-1.63.0.eb'],
+ 'version: 1.64.0; versionsuffix:': ['Boost.Python-1.64.0-gompi-2018a.eb'],
+ 'version: 1.66.0; versionsuffix:': ['BLAST+-2.7.1-foss-2018a.eb'],
+ }))
+
+ self.assertTrue(self.check_dep_vars('2019a', 'Boost', {
+ 'version: 1.64.0; versionsuffix:': [
+ 'Boost.Python-1.64.0-gompi-2019a.eb',
+ 'EMAN2-2.3-foss-2019a-Python-2.7.15.eb',
+ ],
+ 'version: 1.70.0; versionsuffix:': [
+ 'BLAST+-2.9.0-gompi-2019a.eb',
+ 'Boost.Python-1.70.0-gompi-2019a.eb',
+ ],
+ }))
+
+ # two variants is OK, if they're for Python 2.x and 3.x
+ self.assertTrue(self.check_dep_vars('2020a', 'Python', {
+ 'version: 2.7.18; versionsuffix:': ['SciPy-bundle-2020.03-foss-2020a-Python-2.7.18.eb'],
+ 'version: 3.8.2; versionsuffix:': ['SciPy-bundle-2020.03-foss-2020a-Python-3.8.2.eb'],
+ }))
+
+ self.assertTrue(self.check_dep_vars('2020a', 'SciPy-bundle', {
+ 'version: 2020.03; versionsuffix: -Python-2.7.18': ['matplotlib-3.2.1-foss-2020a-Python-2.7.18.eb'],
+ 'version: 2020.03; versionsuffix: -Python-3.8.2': ['matplotlib-3.2.1-foss-2020a-Python-3.8.2.eb'],
+ }))
+
+ # for recent easyconfig generations, there's no versionsuffix anymore for Python 3
+ self.assertTrue(self.check_dep_vars('2020b', 'Python', {
+ 'version: 2.7.18; versionsuffix:': ['SciPy-bundle-2020.11-foss-2020b-Python-2.7.18.eb'],
+ 'version: 3.8.6; versionsuffix:': ['SciPy-bundle-2020.11-foss-2020b.eb'],
+ }))
+
+ self.assertTrue(self.check_dep_vars('GCCcore-10.2', 'PyYAML', {
+ 'version: 5.3.1; versionsuffix:': ['IPython-7.18.1-GCCcore-10.2.0.eb'],
+ 'version: 5.3.1; versionsuffix: -Python-2.7.18': ['IPython-7.18.1-GCCcore-10.2.0-Python-2.7.18.eb'],
+ }))
+
+ self.assertTrue(self.check_dep_vars('2020b', 'SciPy-bundle', {
+ 'version: 2020.11; versionsuffix: -Python-2.7.18': ['matplotlib-3.3.3-foss-2020b-Python-2.7.18.eb'],
+ 'version: 2020.11; versionsuffix:': ['matplotlib-3.3.3-foss-2020b.eb'],
+ }))
+
+ # not allowed for older generations (foss/intel 2020a or older, GCC(core) 10.1.0 or older)
+ self.assertFalse(self.check_dep_vars('2020a', 'SciPy-bundle', {
+ 'version: 2020.03; versionsuffix: -Python-2.7.18': ['matplotlib-3.2.1-foss-2020a-Python-2.7.18.eb'],
+ 'version: 2020.03; versionsuffix:': ['matplotlib-3.2.1-foss-2020a.eb'],
+ }))
+
+ def test_dep_versions_per_toolchain_generation(self):
+ """
+ Check whether there's only one dependency version per toolchain generation actively used.
+ This is enforced to try and limit the chance of running into conflicts when multiple modules built with
+ the same toolchain are loaded together.
+ """
+ ecs_by_full_mod_name = dict((ec['full_mod_name'], ec) for ec in self.parsed_easyconfigs)
+ if len(ecs_by_full_mod_name) != len(self.parsed_easyconfigs):
+ self.fail('Easyconfigs with duplicate full_mod_name found')
+
+ # Cache already determined dependencies
+ ec_to_deps = dict()
+
+ def get_deps_for(ec):
+ """Get list of (direct) dependencies for specified easyconfig."""
+ ec_mod_name = ec['full_mod_name']
+ deps = ec_to_deps.get(ec_mod_name)
+ if deps is None:
+ deps = []
+ for dep in ec['ec']['dependencies']:
+ dep_mod_name = dep['full_mod_name']
+ deps.append((dep['name'], dep['version'], dep['versionsuffix'], dep_mod_name))
+ # Note: Raises KeyError if dep not found
+ res = ecs_by_full_mod_name[dep_mod_name]
+ deps.extend(get_deps_for(res))
+ ec_to_deps[ec_mod_name] = deps
+
+ return deps
+
+ # some software also follows {a,b} versioning scheme,
+ # which throws off the pattern matching done below for toolchain versions
+ false_positives_regex = re.compile('^MATLAB-Engine-20[0-9][0-9][ab]')
+
+ # restrict to checking dependencies of easyconfigs using common toolchains (start with 2018a)
+ # and GCCcore subtoolchain for common toolchains, starting with GCCcore 7.x
+ for pattern in ['20(1[89]|[2-9][0-9])[ab]', r'GCCcore-([7-9]|[1-9][0-9])\.[0-9]']:
+ all_deps = {}
+ regex = re.compile(r'^.*-(?P%s).*\.eb$' % pattern)
+
+ # collect variants for all dependencies of easyconfigs that use a toolchain that matches
+ for ec in self.ordered_specs:
+ ec_file = os.path.basename(ec['spec'])
+
+ # take into account software which also follows a {a,b} versioning scheme
+ ec_file = false_positives_regex.sub('', ec_file)
+
+ res = regex.match(ec_file)
+ if res:
+ tc_gen = res.group('tc_gen')
+ all_deps_tc_gen = all_deps.setdefault(tc_gen, {})
+ for dep_name, dep_ver, dep_versuff, dep_mod_name in get_deps_for(ec):
+ dep_variants = all_deps_tc_gen.setdefault(dep_name, {})
+ # a variant is defined by version + versionsuffix
+ variant = "version: %s; versionsuffix: %s" % (dep_ver, dep_versuff)
+ # keep track of which easyconfig this is a dependency
+ dep_variants.setdefault(variant, set()).add(ec_file)
+
+ # check which dependencies have more than 1 variant
+ multi_dep_vars, multi_dep_vars_msg = [], ''
+ for tc_gen in sorted(all_deps.keys()):
+ for dep in sorted(all_deps[tc_gen].keys()):
+ dep_vars = all_deps[tc_gen][dep]
+ if not self.check_dep_vars(tc_gen, dep, dep_vars):
+ multi_dep_vars.append(dep)
+ multi_dep_vars_msg += "\nfound %s variants of '%s' dependency " % (len(dep_vars), dep)
+ multi_dep_vars_msg += "in easyconfigs using '%s' toolchain generation\n* " % tc_gen
+ multi_dep_vars_msg += '\n* '.join("%s as dep for %s" % v for v in sorted(dep_vars.items()))
+ multi_dep_vars_msg += '\n'
+
+ error_msg = "No multi-variant deps found for '%s' easyconfigs:\n%s" % (regex.pattern, multi_dep_vars_msg)
+ self.assertFalse(multi_dep_vars, error_msg)
+
+ def test_sanity_check_paths(self):
+ """Make sure specified sanity check paths adher to the requirements."""
+
+ for ec in self.parsed_easyconfigs:
+ ec_scp = ec['ec']['sanity_check_paths']
+ if ec_scp != {}:
+ # if sanity_check_paths is specified (i.e., non-default), it must adher to the requirements
+ # both 'files' and 'dirs' keys, both with list values and with at least one a non-empty list
+ error_msg = "sanity_check_paths for %s does not meet requirements: %s" % (ec['spec'], ec_scp)
+ self.assertEqual(sorted(ec_scp.keys()), ['dirs', 'files'], error_msg)
+ self.assertTrue(isinstance(ec_scp['dirs'], list), error_msg)
+ self.assertTrue(isinstance(ec_scp['files'], list), error_msg)
+ self.assertTrue(ec_scp['dirs'] or ec_scp['files'], error_msg)
+
+ def test_r_libs_site_env_var(self):
+ """Make sure $R_LIBS_SITE is being updated, rather than $R_LIBS."""
+ # cfr. https://github.com/easybuilders/easybuild-easyblocks/pull/2326
+
+ r_libs_ecs = []
+ for ec in self.parsed_easyconfigs:
+ for key in ('modextrapaths', 'modextravars'):
+ if 'R_LIBS' in ec['ec'][key]:
+ r_libs_ecs.append(ec['spec'])
+
+ error_msg = "%d easyconfigs found which set $R_LIBS, should be $R_LIBS_SITE: %s"
+ self.assertEqual(r_libs_ecs, [], error_msg % (len(r_libs_ecs), ', '.join(r_libs_ecs)))
+
+ def test_easyconfig_locations(self):
+ """Make sure all easyconfigs files are in the right location."""
+ easyconfig_dirs_regex = re.compile(r'/easybuild/easyconfigs/[0a-z]/[^/]+$')
+ topdir = os.path.dirname(os.path.dirname(os.path.dirname(__file__)))
+ for (dirpath, _, filenames) in os.walk(topdir):
+ # ignore git/svn dirs & archived easyconfigs
+ if '/.git/' in dirpath or '/.svn/' in dirpath or '__archive__' in dirpath:
+ continue
+ # check whether list of .eb files is non-empty
+ easyconfig_files = [fn for fn in filenames if fn.endswith('eb')]
+ if easyconfig_files:
+ # check whether path matches required pattern
+ if not easyconfig_dirs_regex.search(dirpath):
+ # only exception: TEMPLATE.eb
+ if not (dirpath.endswith('/easybuild/easyconfigs') and filenames == ['TEMPLATE.eb']):
+ self.assertTrue(False, "List of easyconfig files in %s is empty: %s" % (dirpath, filenames))
+
+ @skip_if_not_pr_to_non_main_branch()
+ def test_pr_sha256_checksums(self):
+ """Make sure changed easyconfigs have SHA256 checksums in place."""
+
+ # list of software for which checksums can not be required,
+ # e.g. because 'source' files need to be constructed manually
+ whitelist = [
+ 'Kent_tools-*',
+ 'MATLAB-*',
+ 'OCaml-*',
+ 'OpenFOAM-Extend-4.1-*',
+ # sources for old versions of Bioconductor packages are no longer available,
+ # so not worth adding checksums for at this point
+ 'R-bundle-Bioconductor-3.[2-5]',
+ ]
+
+ # the check_sha256_checksums function (again) creates an EasyBlock instance
+ # for easyconfigs using the Bundle easyblock, this is a problem because the 'sources' easyconfig parameter
+ # is updated in place (sources for components are added to the 'parent' sources) in Bundle's __init__;
+ # therefore, we need to reset 'sources' to an empty list here if Bundle is used...
+ # likewise for 'patches' and 'checksums'
+ for ec in self.changed_ecs:
+ if ec['easyblock'] in ['Bundle', 'PythonBundle', 'EB_OpenSSL_wrapper'] or ec['name'] in ['Clang-AOMP']:
+ ec['sources'] = []
+ ec['patches'] = []
+ ec['checksums'] = []
+
+ # filter out deprecated easyconfigs
+ retained_changed_ecs = []
+ for ec in self.changed_ecs:
+ if not ec['deprecated']:
+ retained_changed_ecs.append(ec)
+
+ checksum_issues = check_sha256_checksums(retained_changed_ecs, whitelist=whitelist)
+ self.assertTrue(len(checksum_issues) == 0, "No checksum issues:\n%s" % '\n'.join(checksum_issues))
+
+ @skip_if_not_pr_to_non_main_branch()
+ def test_pr_python_packages(self):
+ """Several checks for easyconfigs that install (bundles of) Python packages."""
+
+ # These packages do not support installation with 'pip'
+ whitelist_pip = [
+ r'ESMPy-.*',
+ r'MATLAB-Engine-.*',
+ r'Meld-.*',
+ r'PyTorch-.*',
+ ]
+
+ whitelist_pip_check = [
+ r'Mako-1.0.4.*Python-2.7.12.*',
+ # no pip 9.x or newer for configparser easyconfigs using a 2016a or 2016b toolchain
+ r'configparser-3.5.0.*-2016[ab].*',
+ # mympirun is installed with system Python, pip may not be installed for system Python
+ r'vsc-mympirun.*',
+ ]
+
+ failing_checks = []
+
+ python_default_urls = PythonPackage.extra_options()['source_urls'][0]
+
+ for ec in self.changed_ecs:
+
+ with ec.disable_templating():
+ ec_fn = os.path.basename(ec.path)
+ easyblock = ec.get('easyblock')
+ exts_defaultclass = ec.get('exts_defaultclass')
+ exts_default_options = ec.get('exts_default_options', {})
+
+ download_dep_fail = ec.get('download_dep_fail')
+ exts_download_dep_fail = ec.get('exts_download_dep_fail')
+ use_pip = ec.get('use_pip')
+ if use_pip is None:
+ use_pip = exts_default_options.get('use_pip')
+
+ # only easyconfig parameters as they are defined in the easyconfig file,
+ # does *not* include other easyconfig parameters with their default value!
+ pure_ec = ec.parser.get_config_dict()
+
+ # download_dep_fail should be set when using PythonPackage
+ if easyblock == 'PythonPackage':
+ if download_dep_fail is None:
+ failing_checks.append("'download_dep_fail' should be set in %s" % ec_fn)
+
+ if pure_ec.get('source_urls') == python_default_urls:
+ failing_checks.append("'source_urls' should not be defined when using the default value "
+ "in %s" % ec_fn)
+
+ # use_pip should be set when using PythonPackage or PythonBundle (except for whitelisted easyconfigs)
+ if easyblock in ['PythonBundle', 'PythonPackage']:
+ if use_pip is None and not any(re.match(regex, ec_fn) for regex in whitelist_pip):
+ failing_checks.append("'use_pip' should be set in %s" % ec_fn)
+
+ # download_dep_fail is enabled automatically in PythonBundle easyblock, so shouldn't be set
+ if easyblock == 'PythonBundle':
+ if download_dep_fail or exts_download_dep_fail:
+ fail = "'*download_dep_fail' should not be set in %s since PythonBundle easyblock is used" % ec_fn
+ failing_checks.append(fail)
+ if pure_ec.get('exts_default_options', {}).get('source_urls') == python_default_urls:
+ failing_checks.append("'source_urls' should not be defined in exts_default_options when using "
+ "the default value in %s" % ec_fn)
+
+ elif exts_defaultclass == 'PythonPackage':
+ # bundle of Python packages should use PythonBundle
+ if easyblock == 'Bundle':
+ fail = "'PythonBundle' easyblock should be used for bundle of Python packages in %s" % ec_fn
+ failing_checks.append(fail)
+ else:
+ # both download_dep_fail and use_pip should be set via exts_default_options
+ # when installing Python packages as extensions
+ for key in ['download_dep_fail', 'use_pip']:
+ if exts_default_options.get(key) is None:
+ failing_checks.append("'%s' should be set in exts_default_options in %s" % (key, ec_fn))
+
+ # if Python is a dependency, that should be reflected in the versionsuffix
+ # Tkinter is an exception, since its version always matches the Python version anyway
+ # Python 3.8.6 and later are also excluded, as we consider python 3 the default python
+ # Also whitelist some updated versions of Amber
+ whitelist_python_suffix = [
+ 'Amber-16-*-2018b-AmberTools-17-patchlevel-10-15.eb',
+ 'Amber-16-intel-2017b-AmberTools-17-patchlevel-8-12.eb',
+ 'R-keras-2.1.6-foss-2018a-R-3.4.4.eb',
+ ]
+ whitelisted = any(re.match(regex, ec_fn) for regex in whitelist_python_suffix)
+ has_python_dep = any(LooseVersion(dep['version']) < LooseVersion('3.8.6')
+ for dep in ec['dependencies'] if dep['name'] == 'Python')
+ if has_python_dep and ec.name != 'Tkinter' and not whitelisted:
+ if not re.search(r'-Python-[23]\.[0-9]+\.[0-9]+', ec['versionsuffix']):
+ msg = "'-Python-%%(pyver)s' should be included in versionsuffix in %s" % ec_fn
+ # This is only a failure for newly added ECs, not for existing ECS
+ # As that would probably break many ECs
+ if ec_fn in self.added_ecs_filenames:
+ failing_checks.append(msg)
+ else:
+ print('\nNote: Failed non-critical check: ' + msg)
+ else:
+ has_recent_python3_dep = any(LooseVersion(dep['version']) >= LooseVersion('3.8.6')
+ for dep in ec['dependencies'] if dep['name'] == 'Python')
+ if has_recent_python3_dep and re.search(r'-Python-3\.[0-9]+\.[0-9]+', ec['versionsuffix']):
+ msg = "'-Python-%%(pyver)s' should no longer be included in versionsuffix in %s" % ec_fn
+ failing_checks.append(msg)
+
+ # require that running of "pip check" during sanity check is enabled via sanity_pip_check
+ if easyblock in ['PythonBundle', 'PythonPackage']:
+ sanity_pip_check = ec.get('sanity_pip_check') or exts_default_options.get('sanity_pip_check')
+ if not sanity_pip_check and not any(re.match(regex, ec_fn) for regex in whitelist_pip_check):
+ failing_checks.append("sanity_pip_check should be enabled in %s" % ec_fn)
+
+ if failing_checks:
+ self.fail('\n'.join(failing_checks))
+
+ @skip_if_not_pr_to_non_main_branch()
+ def test_pr_R_packages(self):
+ """Several checks for easyconfigs that install (bundles of) R packages."""
+ failing_checks = []
+
+ for ec in self.changed_ecs:
+ ec_fn = os.path.basename(ec.path)
+ exts_defaultclass = ec.get('exts_defaultclass')
+ if exts_defaultclass == 'RPackage' or ec.name == 'R':
+ seen_exts = set()
+ for ext in ec['exts_list']:
+ if isinstance(ext, (tuple, list)):
+ ext_name = ext[0]
+ else:
+ ext_name = ext
+ if ext_name in seen_exts:
+ failing_checks.append('%s was added multiple times to exts_list in %s' % (ext_name, ec_fn))
+ else:
+ seen_exts.add(ext_name)
+ self.assertFalse(failing_checks, '\n'.join(failing_checks))
+
+ @skip_if_not_pr_to_non_main_branch()
+ def test_pr_sanity_check_paths(self):
+ """Make sure a custom sanity_check_paths value is specified for easyconfigs that use a generic easyblock."""
+
+ # some generic easyblocks already have a decent customised sanity_check_paths,
+ # including CMakePythonPackage, GoPackage, PythonBundle & PythonPackage;
+ # BuildEnv, ModuleRC and Toolchain easyblocks doesn't install anything so there is nothing to check.
+ whitelist = ['BuildEnv', 'CMakePythonPackage', 'CrayToolchain', 'GoPackage', 'ModuleRC',
+ 'PythonBundle', 'PythonPackage', 'Toolchain']
+ # Bundles of dependencies without files of their own
+ # Autotools: Autoconf + Automake + libtool, (recent) GCC: GCCcore + binutils, CUDA: GCC + CUDAcore,
+ # CESM-deps: Python + Perl + netCDF + ESMF + git, FEniCS: DOLFIN and co
+ bundles_whitelist = ['Autotools', 'CESM-deps', 'CUDA', 'GCC', 'FEniCS', 'ESL-Bundle', 'ROCm']
+
+ failing_checks = []
+
+ for ec in self.changed_ecs:
+ easyblock = ec.get('easyblock')
+ if is_generic_easyblock(easyblock) and not ec.get('sanity_check_paths'):
+
+ sanity_check_ok = False
+
+ if easyblock in whitelist or (easyblock == 'Bundle' and ec['name'] in bundles_whitelist):
+ sanity_check_ok = True
+
+ # also allow bundles that enable per-component sanity checks
+ elif easyblock == 'Bundle':
+ if ec['sanity_check_components'] or ec['sanity_check_all_components']:
+ sanity_check_ok = True
+
+ if not sanity_check_ok:
+ ec_fn = os.path.basename(ec.path)
+ failing_checks.append("No custom sanity_check_paths found in %s" % ec_fn)
+
+ self.assertFalse(failing_checks, '\n'.join(failing_checks))
+
+ @skip_if_not_pr_to_non_main_branch()
+ def test_pr_https(self):
+ """Make sure https:// URL is used (if it exists) for homepage/source_urls (rather than http://)."""
+
+ whitelist = [
+ 'Kaiju', # invalid certificate at https://kaiju.binf.ku.dk
+ 'libxml2', # https://xmlsoft.org works, but invalid certificate
+ 'p4vasp', # https://www.p4vasp.at doesn't work
+ 'ITSTool', # https://itstool.org/ doesn't work
+ 'UCX-', # bad certificate for https://www.openucx.org
+ 'MUMPS', # https://mumps.enseeiht.fr doesn't work
+ 'PyFR', # https://www.pyfr.org doesn't work
+ 'PycURL', # bad certificate for https://pycurl.io/
+ ]
+ url_whitelist = [
+ # https:// doesn't work, results in index page being downloaded instead
+ # (see https://github.com/easybuilders/easybuild-easyconfigs/issues/9692)
+ 'http://isl.gforge.inria.fr',
+ # https:// leads to File Not Found
+ 'http://tau.uoregon.edu/',
+ # https:// has outdated SSL configurations
+ 'http://faculty.scs.illinois.edu',
+ ]
+ # Cache: Mapping of already checked HTTP urls to whether the HTTPS variant works
+ checked_urls = dict()
+
+ def check_https_url(http_url):
+ """Check if the https url works"""
+ http_url = http_url.rstrip('/') # Remove trailing slashes
+ https_url_works = checked_urls.get(http_url)
+ if https_url_works is None:
+ https_url = http_url.replace('http://', 'https://')
+ try:
+ https_url_works = bool(urlopen(https_url, timeout=5))
+ except Exception:
+ https_url_works = False
+ checked_urls[http_url] = https_url_works
+
+ http_regex = re.compile('http://[^"\'\n]+', re.M)
+
+ failing_checks = []
+ for ec in self.changed_ecs:
+ ec_fn = os.path.basename(ec.path)
+
+ # skip whitelisted easyconfigs
+ if any(ec_fn.startswith(x) for x in whitelist):
+ continue
+
+ # ignore commented out lines in easyconfig files when checking for http:// URLs
+ ec_txt = '\n'.join(line for line in ec.rawtxt.split('\n') if not line.startswith('#'))
+
+ for http_url in http_regex.findall(ec_txt):
+
+ # skip whitelisted http:// URLs
+ if any(http_url.startswith(x) for x in url_whitelist):
+ continue
+
+ if check_https_url(http_url):
+ failing_checks.append("Found http:// URL in %s, should be https:// : %s" % (ec_fn, http_url))
+ if failing_checks:
+ self.fail('\n'.join(failing_checks))
+
+ @skip_if_not_pr_to_non_main_branch()
+ def test_pr_patch_descr(self):
+ """
+ Check whether all patch files touched in PR have a description on top.
+ """
+ no_descr_patches = []
+ for patch in self.changed_patches:
+ patch_txt = read_file(patch)
+ if patch_txt.startswith('--- '):
+ no_descr_patches.append(patch)
+
+ self.assertFalse(no_descr_patches, "No description found in patches: %s" % ', '.join(no_descr_patches))
+
+
+def template_easyconfig_test(self, spec):
+ """Tests for an individual easyconfig: parsing, instantiating easyblock, check patches, ..."""
+
+ # set to False, so it's False in case of this test failing
+ global single_tests_ok
+ prev_single_tests_ok = single_tests_ok
+ single_tests_ok = False
+
+ # parse easyconfig
+ ecs = process_easyconfig(spec)
+ if len(ecs) == 1:
+ ec = ecs[0]['ec']
+
+ # cache the parsed easyconfig, to avoid that it is parsed again
+ EasyConfigTest._parsed_easyconfigs.append(ecs[0])
+ else:
+ self.assertTrue(False, "easyconfig %s does not contain blocks, yields only one parsed easyconfig" % spec)
+
+ # check easyconfig file name
+ expected_fn = '%s-%s.eb' % (ec['name'], det_full_ec_version(ec))
+ msg = "Filename '%s' of parsed easyconfig matches expected filename '%s'" % (spec, expected_fn)
+ self.assertEqual(os.path.basename(spec), expected_fn, msg)
+
+ name, easyblock = fetch_parameters_from_easyconfig(ec.rawtxt, ['name', 'easyblock'])
+
+ # make sure easyconfig file is in expected location
+ expected_subdir = os.path.join('easybuild', 'easyconfigs', letter_dir_for(name), name)
+ subdir = os.path.join(*spec.split(os.path.sep)[-5:-1])
+ fail_msg = "Easyconfig file %s not in expected subdirectory %s" % (spec, expected_subdir)
+ self.assertEqual(expected_subdir, subdir, fail_msg)
+
+ # sanity check for software name, moduleclass
+ self.assertEqual(ec['name'], name)
+ self.assertTrue(ec['moduleclass'] in build_option('valid_module_classes'))
+
+ # instantiate easyblock with easyconfig file
+ app_class = get_easyblock_class(easyblock, name=name)
+
+ # check that automagic fallback to ConfigureMake isn't done (deprecated behaviour)
+ fn = os.path.basename(spec)
+ error_msg = "%s relies on automagic fallback to ConfigureMake, should use easyblock = 'ConfigureMake' instead" % fn
+ self.assertTrue(easyblock or app_class is not ConfigureMake, error_msg)
+
+ # dump the easyconfig file;
+ # this should be done before creating the easyblock instance (done below via app_class),
+ # because some easyblocks (like PythonBundle) modify easyconfig parameters at initialisation
+ handle, test_ecfile = tempfile.mkstemp()
+ os.close(handle)
+
+ ec.dump(test_ecfile)
+ dumped_ec = EasyConfigParser(test_ecfile).get_config_dict()
+ os.remove(test_ecfile)
+
+ app = app_class(ec)
+
+ # more sanity checks
+ self.assertTrue(name, app.name)
+ self.assertTrue(ec['version'], app.version)
+
+ # make sure that deprecated 'dummy' toolchain is no longer used, should use 'system' toolchain instead
+ ec_fn = os.path.basename(spec)
+ error_msg_tmpl = "%s should use 'system' toolchain rather than deprecated 'dummy' toolchain"
+ self.assertFalse(ec['toolchain']['name'] == 'dummy', error_msg_tmpl % os.path.basename(spec))
+
+ # make sure that $root is not used, since it is not compatible with module files in Lua syntax
+ res = re.findall(r'.*\$root.*', ec.rawtxt, re.M)
+ error_msg = "Found use of '$root', not compatible with modules in Lua syntax, use '%%(installdir)s' instead: %s"
+ self.assertFalse(res, error_msg % res)
+
+ # check for redefined easyconfig parameters, there should be none...
+ param_def_regex = re.compile(r'^(?P\w+)\s*=', re.M)
+ keys = param_def_regex.findall(ec.rawtxt)
+ redefined_keys = []
+ for key in sorted(nub(keys)):
+ cnt = keys.count(key)
+ if cnt > 1:
+ redefined_keys.append((key, cnt))
+
+ redefined_keys_error_msg = "There should be no redefined easyconfig parameters, found %d: " % len(redefined_keys)
+ redefined_keys_error_msg += ', '.join('%s (%d)' % x for x in redefined_keys)
+
+ self.assertFalse(redefined_keys, redefined_keys_error_msg)
+
+ # make sure old GitHub urls for EasyBuild that include 'hpcugent' are no longer used
+ old_urls = [
+ 'github.com/hpcugent/easybuild',
+ 'hpcugent.github.com/easybuild',
+ 'hpcugent.github.io/easybuild',
+ ]
+ for old_url in old_urls:
+ self.assertFalse(old_url in ec.rawtxt, "Old URL '%s' not found in %s" % (old_url, spec))
+
+ # make sure binutils is included as a (build) dep if toolchain is GCCcore
+ if ec['toolchain']['name'] == 'GCCcore':
+ # easyblocks without a build step
+ non_build_blocks = ['Binary', 'JAR', 'PackedBinary', 'Tarball']
+ # some software packages do not have a build step
+ non_build_soft = ['ANIcalculator', 'Eigen']
+
+ requires_binutils = ec['easyblock'] not in non_build_blocks and ec['name'] not in non_build_soft
+
+ # let's also exclude the very special case where the system GCC is used as GCCcore, and only apply this
+ # exception to the dependencies of binutils (since we should eventually build a new binutils with GCCcore)
+ if ec['toolchain']['version'] == 'system':
+ binutils_complete_dependencies = ['M4', 'Bison', 'flex', 'help2man', 'zlib', 'binutils']
+ requires_binutils &= bool(ec['name'] not in binutils_complete_dependencies)
+
+ # if no sources/extensions/components are specified, it's just a bundle (nothing is being compiled)
+ requires_binutils &= bool(ec['sources'] or ec['exts_list'] or ec.get('components'))
+
+ if requires_binutils:
+ # dependencies() returns both build and runtime dependencies
+ # in some cases, binutils can also be a runtime dep (e.g. for Clang)
+ # Also using GCC directly as a build dep is also allowed (it includes the correct binutils)
+ dep_names = [d['name'] for d in ec.dependencies()]
+ self.assertTrue('binutils' in dep_names or 'GCC' in dep_names,
+ "binutils or GCC is a build dep in %s: %s" % (spec, dep_names))
+
+ # make sure that OpenSSL wrapper is used rather than OS dependency,
+ # for easyconfigs using a 2021a (sub)toolchain or more recent common toolchain version
+ osdeps = ec['osdependencies']
+ if osdeps:
+ # check whether any entry in osdependencies related to OpenSSL
+ openssl_osdep = False
+ for osdep in osdeps:
+ if isinstance(osdep, string_type):
+ osdep = [osdep]
+ if any('libssl' in x for x in osdep) or any('openssl' in x for x in osdep):
+ openssl_osdep = True
+
+ if openssl_osdep:
+ tcname = ec['toolchain']['name']
+ tcver = LooseVersion(ec['toolchain']['version'])
+
+ gcc_subtc_2021a = tcname in ('GCCcore', 'GCC') and tcver > LooseVersion('10.3')
+ if gcc_subtc_2021a or (tcname in ('foss', 'gompi', 'iimpi', 'intel') and tcver >= LooseVersion('2021')):
+ self.assertFalse(openssl_osdep, "OpenSSL should not be listed as OS dependency in %s" % spec)
+
+ src_cnt = len(ec['sources'])
+ patch_checksums = ec['checksums'][src_cnt:]
+
+ # make sure all patch files are available
+ specdir = os.path.dirname(spec)
+ specfn = os.path.basename(spec)
+ for idx, patch in enumerate(ec['patches']):
+ if isinstance(patch, (tuple, list)):
+ patch = patch[0]
+
+ # only check actual patch files, not other files being copied via the patch functionality
+ patch_full = os.path.join(specdir, patch)
+ if patch.endswith('.patch'):
+ msg = "Patch file %s is available for %s" % (patch_full, specfn)
+ self.assertTrue(os.path.isfile(patch_full), msg)
+
+ # verify checksum for each patch file
+ if idx < len(patch_checksums) and (os.path.exists(patch_full) or patch.endswith('.patch')):
+ checksum = patch_checksums[idx]
+ error_msg = "Invalid checksum for patch file %s in %s: %s" % (patch, ec_fn, checksum)
+ res = verify_checksum(patch_full, checksum)
+ self.assertTrue(res, error_msg)
+
+ # make sure 'source' step is not being skipped,
+ # since that implies not verifying the checksum
+ error_msg = "'source' step should not be skipped in %s, since that implies not verifying checksums" % ec_fn
+ self.assertFalse(ec['checksums'] and ('source' in ec['skipsteps']), error_msg)
+
+ for ext in ec.get_ref('exts_list'):
+ if isinstance(ext, (tuple, list)) and len(ext) == 3:
+ ext_name = ext[0]
+ self.assertTrue(isinstance(ext[2], dict),
+ "3rd element of extension spec for %s must be a dictionary" % ext_name)
+
+ # After the sanity check above, use collect_exts_file_info to resolve templates etc. correctly
+ for ext in app.collect_exts_file_info(fetch_files=False, verify_checksums=False):
+ try:
+ ext_options = ext['options']
+ except KeyError:
+ # No options --> Only have a name which is valid, so nothing to check
+ continue
+
+ checksums = ext_options.get('checksums', [])
+ src_cnt = len(ext_options.get('sources', [])) or 1
+ patch_checksums = checksums[src_cnt:]
+
+ for idx, ext_patch in enumerate(ext.get('patches', [])):
+ if isinstance(ext_patch, (tuple, list)):
+ ext_patch = ext_patch[0]
+
+ # only check actual patch files, not other files being copied via the patch functionality
+ ext_patch_full = os.path.join(specdir, ext_patch['name'])
+ if ext_patch_full.endswith('.patch'):
+ msg = "Patch file %s is available for %s" % (ext_patch_full, specfn)
+ self.assertTrue(os.path.isfile(ext_patch_full), msg)
+
+ # verify checksum for each patch file
+ if idx < len(patch_checksums) and (os.path.exists(ext_patch_full) or ext_patch.endswith('.patch')):
+ checksum = patch_checksums[idx]
+ error_msg = "Invalid checksum for patch %s for %s extension in %s: %s"
+ res = verify_checksum(ext_patch_full, checksum)
+ self.assertTrue(res, error_msg % (ext_patch, ext_name, ec_fn, checksum))
+
+ # check whether all extra_options defined for used easyblock are defined
+ extra_opts = app.extra_options()
+ for key in extra_opts:
+ self.assertTrue(key in app.cfg)
+
+ app.close_log()
+ os.remove(app.logfile)
+
+ # inject dummy values for templates that are only known at a later stage
+ dummy_template_values = {
+ 'builddir': '/dummy/builddir',
+ 'installdir': '/dummy/installdir',
+ 'parallel': '2',
+ }
+ ec.template_values.update(dummy_template_values)
+
+ ec_dict = ec.parser.get_config_dict()
+ orig_toolchain = ec_dict['toolchain']
+ for key in ec_dict:
+ # skip parameters for which value is equal to default value
+ orig_val = ec_dict[key]
+ if key in DEFAULT_CONFIG and orig_val == DEFAULT_CONFIG[key][0]:
+ continue
+ if key in extra_opts and orig_val == extra_opts[key][0]:
+ continue
+ if key not in DEFAULT_CONFIG and key not in extra_opts:
+ continue
+
+ orig_val = resolve_template(ec_dict[key], ec.template_values)
+ dumped_val = resolve_template(dumped_ec[key], ec.template_values)
+
+ # take into account that dumped value for *dependencies may include hard-coded subtoolchains
+ # if no easyconfig was found for the dependency with the 'parent' toolchain,
+ # if may get resolved using a subtoolchain, which is then hardcoded in the dumped easyconfig
+ if key in DEPENDENCY_PARAMETERS:
+ # number of dependencies should remain the same
+ self.assertEqual(len(orig_val), len(dumped_val))
+ for orig_dep, dumped_dep in zip(orig_val, dumped_val):
+ # name should always match
+ self.assertEqual(orig_dep[0], dumped_dep[0])
+
+ # version should always match, or be a possibility from the version dict
+ if isinstance(orig_dep[1], dict):
+ self.assertTrue(dumped_dep[1] in orig_dep[1].values())
+ else:
+ self.assertEqual(orig_dep[1], dumped_dep[1])
+
+ # 3rd value is versionsuffix;
+ if len(dumped_dep) >= 3:
+ # if no versionsuffix was specified in original dep spec, then dumped value should be empty string
+ if len(orig_dep) >= 3:
+ self.assertEqual(dumped_dep[2], orig_dep[2])
+ else:
+ self.assertEqual(dumped_dep[2], '')
+
+ # 4th value is toolchain spec
+ if len(dumped_dep) >= 4:
+ if len(orig_dep) >= 4:
+ # if True was used to indicate that dependency should use system toolchain,
+ # then we need to compare the value for the dumped easyconfig more carefully;
+ # see also https://github.com/easybuilders/easybuild-framework/pull/4069
+ if orig_dep[3] is True:
+ self.assertEqual(dumped_dep[3], EASYCONFIG_CONSTANTS['SYSTEM'][0])
+ else:
+ self.assertEqual(dumped_dep[3], orig_dep[3])
+ else:
+ # if a subtoolchain is specifed (only) in the dumped easyconfig,
+ # it should *not* be the same as the parent toolchain
+ self.assertNotEqual(dumped_dep[3], (orig_toolchain['name'], orig_toolchain['version']))
+
+ # take into account that for some string-valued easyconfig parameters (configopts & co),
+ # the easyblock may have injected additional values, which affects the dumped easyconfig file
+ elif isinstance(orig_val, string_type):
+ error_msg = "%s value '%s' should start with '%s'" % (key, dumped_val, orig_val)
+ self.assertTrue(dumped_val.startswith(orig_val), error_msg)
+ else:
+ error_msg = "%s value should be equal in original and dumped easyconfig: '%s' vs '%s'"
+ self.assertEqual(orig_val, dumped_val, error_msg % (key, orig_val, dumped_val))
+
+ # test passed, so set back to True
+ single_tests_ok = True and prev_single_tests_ok
+
+
+def suite(loader=None):
+ """Return all easyblock initialisation tests."""
+ def make_inner_test(spec_path):
+ def innertest(self):
+ template_easyconfig_test(self, spec_path)
+ return innertest
+
+ # dynamically generate a separate test for each of the available easyconfigs
+ # define new inner functions that can be added as class methods to InitTest
+ easyconfigs_path = get_paths_for('easyconfigs')[0]
+ cnt = 0
+ for (subpath, dirs, specs) in os.walk(easyconfigs_path, topdown=True):
+
+ # ignore archived easyconfigs
+ if '__archive__' in dirs:
+ dirs.remove('__archive__')
+
+ for spec in specs:
+ if spec.endswith('.eb') and spec != 'TEMPLATE.eb':
+ cnt += 1
+ innertest = make_inner_test(os.path.join(subpath, spec))
+ innertest.__doc__ = "Test for easyconfig %s" % spec
+ # double underscore so parsing tests are run first
+ innertest.__name__ = "test__parse_easyconfig_%s" % spec
+ setattr(EasyConfigTest, innertest.__name__, innertest)
+
+ print("Found %s easyconfigs..." % cnt)
+ if not loader:
+ loader = TestLoader()
+ return loader.loadTestsFromTestCase(EasyConfigTest)
+
+
+if __name__ == '__main__':
+ main()