From 89d52bba2b1ded816b2c33c816a8513410220e2b Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Mon, 23 Oct 2023 11:18:18 +0200 Subject: [PATCH 1/4] Add test for duplicate `PYTHONPATH` in `modextrapaths` And similar test for `sanity_check_paths`. Intended to avoid issues fixed in https://github.com/easybuilders/easybuild-easyconfigs/pull/13385 --- test/easyconfigs/easyconfigs.py | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/test/easyconfigs/easyconfigs.py b/test/easyconfigs/easyconfigs.py index 229edaedadae..7dc46f5b8b64 100644 --- a/test/easyconfigs/easyconfigs.py +++ b/test/easyconfigs/easyconfigs.py @@ -1409,6 +1409,29 @@ def test_pr_python_packages(self): failing_checks.append('Use Cargo%s instead of %s when Rust is used in %s' % (easyblock, easyblock, ec_fn)) + # Avoid duplicated PYTHONPATH entries and checks (set by EasyBlock in framework) + default_dirs = [ + 'lib/python%(pyshortver)s/site-packages', + ] + with ec.allow_unresolved_templates(): + default_dirs += [ec.resolve_template(d) for d in default_dirs] + extra_python_path = ec.get('modextrapaths', dict()).get('PYTHONPATH') + failing_checks.extend(f"modextrapaths should not contain {d} (automatically added by EasyBlock)" + for d in default_dirs if d in extra_python_path) + if easyblock == 'PythonBundle' or easyblock.endswith('PythonPackage'): + sanity_check_dirs = ec.get('sanity_check_paths', dict()).get('dirs') or [] + default_dirs = ( + 'lib/python%(pyshortver)s/site-packages', + 'lib/python%(pyshortver)s/site-packages/%(name)s', + 'lib64/python%(pyshortver)s/site-packages', + 'lib64/python%(pyshortver)s/site-packages/%(name)s', + ) + with ec.allow_unresolved_templates(): + default_dirs += [ec.resolve_template(d) for d in default_dirs] + failing_checks.extend(f"sanity_check_paths['dirs'] should not contain {d} " + "(automatically added by easyblock)" + for d in default_dirs if d in sanity_check_dirs) + if failing_checks: self.fail('\n'.join(failing_checks)) From 74583bbeb0cfba28a27993ab49d1b61fa112c109 Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Mon, 23 Oct 2023 12:46:08 +0200 Subject: [PATCH 2/4] Factor out setting name of EC in failing Python checks --- test/easyconfigs/easyconfigs.py | 49 ++++++++++++++++----------------- 1 file changed, 24 insertions(+), 25 deletions(-) diff --git a/test/easyconfigs/easyconfigs.py b/test/easyconfigs/easyconfigs.py index 7dc46f5b8b64..ee80dacae8e1 100644 --- a/test/easyconfigs/easyconfigs.py +++ b/test/easyconfigs/easyconfigs.py @@ -968,7 +968,6 @@ def get_deps_for(ec): '12.2': '2022b', '12.3': '2023a', '13.1': None, - '13.1': None, '13.2': '2023b', '13.3': '2024a', '14.1': None, @@ -1313,6 +1312,7 @@ def test_pr_python_packages(self): python_default_urls = PythonPackage.extra_options()['source_urls'][0] for ec in self.changed_ecs: + failing_checks_ec = [] with ec.disable_templating(): ec_fn = os.path.basename(ec.path) @@ -1332,11 +1332,10 @@ def test_pr_python_packages(self): # download_dep_fail should be set when using PythonPackage if easyblock == 'PythonPackage': if download_dep_fail is None: - failing_checks.append("'download_dep_fail' should be set in %s" % ec_fn) + failing_checks_ec.append("'download_dep_fail' should be set") if pure_ec.get('source_urls') == python_default_urls: - failing_checks.append("'source_urls' should not be defined when using the default value " - "in %s" % ec_fn) + failing_checks_ec.append("'source_urls' should not be defined when using the default value") # --no-build-isolation option for 'pip install' should be enabled pip_no_build_isolation = ec.get('pip_no_build_isolation', True) @@ -1345,19 +1344,19 @@ def test_pr_python_packages(self): ext_opts = ext[2] pip_no_build_isolation &= ext_opts.get('pip_no_build_isolation', True) if not pip_no_build_isolation: - failing_checks.append(f"pip_no_build_isolation should not be disabled in {ec_fn}") + failing_checks_ec.append(f"pip_no_build_isolation should not be disabled") # use_pip should be set when using PythonPackage or PythonBundle, # or an easyblock that derives from it (except for whitelisted easyconfigs) if easyblock in ['CargoPythonBundle', 'CargoPythonPackage', 'PythonBundle', 'PythonPackage']: if use_pip is None and not any(re.match(regex, ec_fn) for regex in whitelist_pip): - failing_checks.append("'use_pip' should be set in %s" % ec_fn) + failing_checks_ec.append("'use_pip' should be set") if exts_defaultclass == 'PythonPackage': # bundle of Python packages should use PythonBundle if easyblock == 'Bundle': - fail = "'PythonBundle' easyblock should be used for bundle of Python packages in %s" % ec_fn - failing_checks.append(fail) + fail = "'PythonBundle' easyblock should be used for bundle of Python packages" + failing_checks_ec.append(fail) # if Python is a dependency, that should be reflected in the versionsuffix since v3.8.6 has_recent_python3_dep = any(LooseVersion(dep['version']) >= LooseVersion('3.8.6') @@ -1370,22 +1369,22 @@ def test_pr_python_packages(self): if ec.name not in exception_python_suffix: if has_old_python_dep and not re.search(r'-Python-[23]\.[0-9]+\.[0-9]+', ec['versionsuffix']): - msg = "'-Python-%%(pyver)s' should be included in versionsuffix in %s" % ec_fn + msg = "'-Python-%(pyver)s' should be included in versionsuffix" # This is only a failure for newly added ECs, not for existing ECS # As that would probably break many ECs if ec_fn in self.added_ecs_filenames: - failing_checks.append(msg) + failing_checks_ec.append(msg) else: - print('\nNote: Failed non-critical check: ' + msg) + print('\nNote: Failed non-critical check for %s: %s' % (ec_fn, msg)) elif has_recent_python3_dep and re.search(r'-Python-3\.[0-9]+\.[0-9]+', ec['versionsuffix']): - msg = "'-Python-%%(pyver)s' should no longer be included in versionsuffix in %s" % ec_fn - failing_checks.append(msg) + msg = "'-Python-%(pyver)s' should no longer be included in versionsuffix" + failing_checks_ec.append(msg) # require that running of "pip check" during sanity check is enabled via sanity_pip_check if easyblock in ['CargoPythonBundle', 'CargoPythonPackage', 'PythonBundle', 'PythonPackage']: sanity_pip_check = ec.get('sanity_pip_check') or exts_default_options.get('sanity_pip_check') if not sanity_pip_check and not any(re.match(regex, ec_fn) for regex in whitelist_pip_check): - failing_checks.append("sanity_pip_check should be enabled in %s" % ec_fn) + failing_checks_ec.append("sanity_pip_check should be enabled") else: # Make sure the user packages in $HOME/.local/lib/python*/ are ignored when running python commands # For the EasyBlocks above this is handled automatically by setting $PYTHONNOUSERSITE @@ -1400,14 +1399,13 @@ def test_pr_python_packages(self): # Strip comment if present to avoid flagging e.g. "sed '/.../' # Fix 'python -c foo' failure" line = comment_re.sub('', line) if python_re.search(line) and not ignore_user_switch_re.search(line): - failing_checks.append("Python invocation in '%s' (line #%s) should use the '-s' parameter in %s" - % (line, line_nr + 1, ec_fn)) + failing_checks_ec.append("Python invocation in '%s' (line #%s) should use the '-s' parameter" + % (line, line_nr + 1)) # When using Rust it should use CargoPython* if easyblock in ('PythonBundle', 'PythonPackage'): if any(dep['name'] == 'Rust' or '-Rust-' in dep['versionsuffix'] for dep in ec.dependencies()): - failing_checks.append('Use Cargo%s instead of %s when Rust is used in %s' - % (easyblock, easyblock, ec_fn)) + failing_checks_ec.append('Use Cargo%s instead of %s when Rust is used' % (easyblock, easyblock)) # Avoid duplicated PYTHONPATH entries and checks (set by EasyBlock in framework) default_dirs = [ @@ -1415,11 +1413,11 @@ def test_pr_python_packages(self): ] with ec.allow_unresolved_templates(): default_dirs += [ec.resolve_template(d) for d in default_dirs] - extra_python_path = ec.get('modextrapaths', dict()).get('PYTHONPATH') - failing_checks.extend(f"modextrapaths should not contain {d} (automatically added by EasyBlock)" - for d in default_dirs if d in extra_python_path) + extra_python_path = ec.get('modextrapaths', {}).get('PYTHONPATH') + failing_checks_ec.extend(f"modextrapaths should not contain {d} (automatically added by EasyBlock)" + for d in default_dirs if d in extra_python_path) if easyblock == 'PythonBundle' or easyblock.endswith('PythonPackage'): - sanity_check_dirs = ec.get('sanity_check_paths', dict()).get('dirs') or [] + sanity_check_dirs = ec.get('sanity_check_paths', {}).get('dirs') or [] default_dirs = ( 'lib/python%(pyshortver)s/site-packages', 'lib/python%(pyshortver)s/site-packages/%(name)s', @@ -1428,9 +1426,10 @@ def test_pr_python_packages(self): ) with ec.allow_unresolved_templates(): default_dirs += [ec.resolve_template(d) for d in default_dirs] - failing_checks.extend(f"sanity_check_paths['dirs'] should not contain {d} " - "(automatically added by easyblock)" - for d in default_dirs if d in sanity_check_dirs) + failing_checks_ec.extend(f"sanity_check_paths['dirs'] should not contain {d} " + "(automatically added by easyblock)" + for d in default_dirs if d in sanity_check_dirs) + failing_checks.extend(f"{ec_fn}: {fail}" for fail in failing_checks_ec) if failing_checks: self.fail('\n'.join(failing_checks)) From be9f4f18199024d803c64cbc734b9d2c8b2d9c22 Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Tue, 14 Oct 2025 10:12:10 +0200 Subject: [PATCH 3/4] Fix type of `default_dirs` --- test/easyconfigs/easyconfigs.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/easyconfigs/easyconfigs.py b/test/easyconfigs/easyconfigs.py index ee80dacae8e1..f8eb19047297 100644 --- a/test/easyconfigs/easyconfigs.py +++ b/test/easyconfigs/easyconfigs.py @@ -1418,12 +1418,12 @@ def test_pr_python_packages(self): for d in default_dirs if d in extra_python_path) if easyblock == 'PythonBundle' or easyblock.endswith('PythonPackage'): sanity_check_dirs = ec.get('sanity_check_paths', {}).get('dirs') or [] - default_dirs = ( + default_dirs = [ 'lib/python%(pyshortver)s/site-packages', 'lib/python%(pyshortver)s/site-packages/%(name)s', 'lib64/python%(pyshortver)s/site-packages', 'lib64/python%(pyshortver)s/site-packages/%(name)s', - ) + ] with ec.allow_unresolved_templates(): default_dirs += [ec.resolve_template(d) for d in default_dirs] failing_checks_ec.extend(f"sanity_check_paths['dirs'] should not contain {d} " From 9fa9549ca59bc3a107486053c3dfced5d1b3cf75 Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Wed, 15 Oct 2025 17:54:54 +0200 Subject: [PATCH 4/4] Fix linter issues --- test/easyconfigs/easyconfigs.py | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/test/easyconfigs/easyconfigs.py b/test/easyconfigs/easyconfigs.py index f8eb19047297..0a1eccfbde64 100644 --- a/test/easyconfigs/easyconfigs.py +++ b/test/easyconfigs/easyconfigs.py @@ -84,9 +84,9 @@ def get_target_branch(): """Return the target branch of a pull request""" # target branch should be anything other than 'master'; # usually is 'develop', but could also be a release branch like '3.7.x' - target_branch = os.environ.get('GITHUB_BASE_REF', None) + target_branch = os.environ.get('GITHUB_BASE_REF') if not target_branch: - target_branch = os.environ.get('TRAVIS_BRANCH', None) + target_branch = os.environ.get('TRAVIS_BRANCH') if not target_branch: raise RuntimeError("Did not find a target branch") return target_branch @@ -1344,7 +1344,7 @@ def test_pr_python_packages(self): ext_opts = ext[2] pip_no_build_isolation &= ext_opts.get('pip_no_build_isolation', True) if not pip_no_build_isolation: - failing_checks_ec.append(f"pip_no_build_isolation should not be disabled") + failing_checks_ec.append("pip_no_build_isolation should not be disabled") # use_pip should be set when using PythonPackage or PythonBundle, # or an easyblock that derives from it (except for whitelisted easyconfigs) @@ -1657,13 +1657,12 @@ def template_easyconfig_test(self, spec): # parse easyconfig ecs = process_easyconfig(spec) - if len(ecs) == 1: - ec = ecs[0]['ec'] + if len(ecs) != 1: + self.fail(f"easyconfig {spec} should not contain blocks and yield only one parsed easyconfig") - # cache the parsed easyconfig, to avoid that it is parsed again - EasyConfigTest._parsed_easyconfigs.append(ecs[0]) - else: - self.fail("easyconfig %s does not contain blocks, yields only one parsed easyconfig" % spec) + ec = ecs[0]['ec'] + # cache the parsed easyconfig, to avoid that it is parsed again + EasyConfigTest._parsed_easyconfigs.append(ecs[0]) # check easyconfig file name expected_fn = '%s-%s.eb' % (ec['name'], det_full_ec_version(ec)) @@ -1781,7 +1780,7 @@ def template_easyconfig_test(self, spec): # exception to the dependencies of binutils (since we should eventually build a new binutils with GCCcore) if ec['toolchain']['version'] == 'system': binutils_complete_dependencies = ['M4', 'Bison', 'flex', 'help2man', 'zlib', 'binutils'] - requires_binutils &= bool(ec['name'] not in binutils_complete_dependencies) + requires_binutils &= ec['name'] not in binutils_complete_dependencies # if no sources/extensions/components are specified, it's just a bundle (nothing is being compiled) requires_binutils &= bool(sources or ec.get_ref('exts_list') or ec.get_ref('components'))