diff --git a/.github/workflows/unit_tests.yml b/.github/workflows/unit_tests.yml index eb1af89f88..b4cd8ff4e6 100644 --- a/.github/workflows/unit_tests.yml +++ b/.github/workflows/unit_tests.yml @@ -9,6 +9,9 @@ concurrency: group: ${{format('{0}:{1}:{2}', github.repository, github.ref, github.workflow)}} cancel-in-progress: true +env: + PYENV_ROOT: /opt/pyenv + jobs: setup: runs-on: ubuntu-latest @@ -22,6 +25,8 @@ jobs: build: needs: setup runs-on: ${{matrix.os || 'ubuntu-24.04'}} + env: + JOB_OS: ${{matrix.os || 'ubuntu-24.04'}} strategy: matrix: # Python 3.10 is default in Ubuntu 22.04 @@ -34,6 +39,9 @@ jobs: - ${{needs.setup.outputs.modules5}} include: # Test different Python 3 versions with Lmod 8.x (with both Lua and Tcl module syntax) + - python: '3.6.1' + modules_tool: ${{needs.setup.outputs.lmod8}} + use_pyenv: '2.6.15' - python: '3.7' modules_tool: ${{needs.setup.outputs.lmod8}} os: ubuntu-22.04 @@ -56,13 +64,7 @@ jobs: steps: - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # 4.2.2 - - name: set up Python - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 - with: - python-version: ${{matrix.python}} - architecture: x64 - - - name: install OS & Python packages + - name: install OS packages run: | # for modules tool APT_PKGS="lua5.3 liblua5.3-dev lua-filesystem lua-posix tcl tcl-dev" @@ -71,6 +73,14 @@ jobs: # dep for GC3Pie APT_PKGS+=" time" + if [[ -n "${{matrix.use_pyenv}}" ]]; then + # See https://github.com/pyenv/pyenv/wiki#suggested-build-environment + APT_PKGS+=" libssl-dev zlib1g-dev \ + libbz2-dev libreadline-dev libsqlite3-dev curl git \ + libncursesw5-dev xz-utils tk-dev libxml2-dev libxmlsec1-dev libffi-dev liblzma-dev" + # Avoid segfault with older Pythons: https://github.com/pyenv/pyenv/issues/2239#issuecomment-1079275184 + APT_PKGS+=" clang" + fi # Avoid apt-get update, as we don't really need it, # and it does more harm than good (it's fairly expensive, and it results in flaky test runs) if ! sudo apt-get install $APT_PKGS; then @@ -79,7 +89,40 @@ jobs: sudo apt-get install $APT_PKGS fi - # Python packages + - name: set up Python + if: '!matrix.use_pyenv' + uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 + with: + python-version: ${{matrix.python}} + architecture: x64 + + - name: Cache PyEnv + id: cache-pyenv + if: matrix.use_pyenv + uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # 4.3.0 + with: + path: ${{env.PYENV_ROOT}} + key: ${{env.JOB_OS}}-py${{matrix.python}}-pyenv${{matrix.use_pyenv}} + + - name: Setup PyEnv + if: matrix.use_pyenv + run: | + if [[ -z "${{steps.cache-pyenv.outputs.cache-hit}}" ]]; then + mkdir "$PYENV_ROOT" + wget https://github.com/pyenv/pyenv/archive/refs/tags/v${{matrix.use_pyenv}}.tar.gz -O- | tar xzf - -C "$PYENV_ROOT" --strip-components=1 + fi + echo "${PYENV_ROOT}/bin" >> $GITHUB_PATH + export PATH="$PYENV_ROOT/bin:$PATH" + echo 'eval "$(pyenv init -)"' > ~/init_pyenv + . ~/init_pyenv + CC=clang pyenv install --skip-existing ${{matrix.python}} + echo 'pyenv global ${{matrix.python}}' >> ~/init_pyenv + + - name: install Python packages + run: | + touch ~/init_pyenv # In case it doesn't exist, make empty file + . ~/init_pyenv + python -V pip --version pip install --upgrade pip pip --version @@ -100,6 +143,7 @@ jobs: # and are only run after the PR gets merged GITHUB_TOKEN: ${{secrets.CI_UNIT_TESTS_GITHUB_TOKEN}} run: | + . ~/init_pyenv # only install GitHub token when testing with Lmod 8.x + Python 3.9, to avoid hitting GitHub rate limit # tests that require a GitHub token are skipped automatically when no GitHub token is available if [[ "${{matrix.modules_tool}}" =~ 'Lmod-8' ]] && [[ "${{matrix.python}}" =~ 3.9 ]]; then @@ -126,6 +170,7 @@ jobs: - name: check sources run: | + . ~/init_pyenv # make sure there are no (top-level) "import setuptools" or "import pkg_resources" statements, # since EasyBuild should not have a runtime requirement on setuptools SETUPTOOLS_IMPORTS=$(egrep --exclude setup.py -RI '^(from|import)[ ]*pkg_resources|^(from|import)[ ]*setuptools' * || true) @@ -133,6 +178,7 @@ jobs: - name: install sources run: | + . ~/init_pyenv # install from source distribution tarball, to test release as published on PyPI python setup.py sdist ls dist @@ -142,7 +188,7 @@ jobs: - name: run test suite env: EB_VERBOSE: 1 - LC_ALL: "" + LC_ALL: "" run: | # run tests *outside* of checked out easybuild-framework directory, # to ensure we're testing installed version (see previous step) @@ -158,7 +204,13 @@ jobs: export PATH=$PREFIX/bin:$(cat $HOME/path) export PYTHONPATH=$PREFIX/lib/python$(echo ${{matrix.python}} | cut -f1,2 -d'.')/site-packages:$PYTHONPATH echo "PYTHONPATH=$PYTHONPATH" - ls $(echo $PYTHONPATH | cut -f1:) + ls $(echo $PYTHONPATH | cut -f1 -d':') + + # Do AFTER setting $PATH above + . ~/init_pyenv + which python3 + python3 --version + python3 -m easybuild.main --version eb --version # tell EasyBuild which modules tool is available diff --git a/easybuild/tools/filetools.py b/easybuild/tools/filetools.py index 5420914b55..5c8f9cfbcc 100644 --- a/easybuild/tools/filetools.py +++ b/easybuild/tools/filetools.py @@ -2632,6 +2632,16 @@ def copy_files(paths, target_path, force_in_dry_run=False, target_single_file=Fa raise EasyBuildError("One or more files to copy should be specified!") +def is_recursive_symlink(path) -> bool: + """Check if the given path is a symlink and points to itself""" + if not os.path.islink(path): + return False + abs_path = os.path.abspath(path) + linkpath = os.path.realpath(abs_path) + abs_path += os.sep # To catch the case where both are equal + return abs_path.startswith(linkpath + os.sep) + + def has_recursive_symlinks(path): """ Check the given directory for recursive symlinks. @@ -2643,12 +2653,9 @@ def has_recursive_symlinks(path): for dirpath, dirnames, filenames in os.walk(path, followlinks=True): for name in itertools.chain(dirnames, filenames): fullpath = os.path.join(dirpath, name) - if os.path.islink(fullpath): - linkpath = os.path.realpath(fullpath) - fullpath += os.sep # To catch the case where both are equal - if fullpath.startswith(linkpath + os.sep): - _log.info("Recursive symlink detected at %s", fullpath) - return True + if is_recursive_symlink(fullpath): + _log.info("Recursive symlink detected at %s", fullpath) + return True return False @@ -2713,7 +2720,20 @@ def copy_dir(path, target_path, force_in_dry_run=False, dirs_exist_ok=False, che else: # if dirs_exist_ok is not enabled or target directory doesn't exist, just use shutil.copytree - shutil.copytree(path, target_path, **kwargs) + if sys.version_info < (3, 7) and kwargs.get('symlinks'): + # Python 3.6 might not correctly detect support for chmod on broken symlinks + # This was fixed in 3.7.0 + # Approach taken from https://bugs.python.org/issue6547: Fail only if all errors are due to symlinks + try: + shutil.copytree(path, target_path, **kwargs) + except shutil.Error as e: + if all(os.path.islink(src) and not os.path.exists(src) or is_recursive_symlink(src) + for src, _dst, _err in e.args[0]): + _log.info("Ignoring errors when copying broken symlinks: %s", e.args[0]) + else: + raise + else: + shutil.copytree(path, target_path, **kwargs) _log.info("%s copied to %s", path, target_path) except (IOError, OSError, shutil.Error) as err: diff --git a/requirements.txt b/requirements.txt index 699cc90372..23c5f26c48 100644 --- a/requirements.txt +++ b/requirements.txt @@ -17,6 +17,12 @@ python-graph-dot python-hglib requests -archspec - +# Some packages dropped support for Python 3.6, so specify max versions +rich<13; python_version < '3.7' rich + +# Similar for dependencies of other packages +cryptography<37; python_version < '3.7' +PyNaCl<1.5; python_version < '3.7' + +archspec diff --git a/test/framework/config.py b/test/framework/config.py index 23ccadb0bc..10614bdfb8 100644 --- a/test/framework/config.py +++ b/test/framework/config.py @@ -59,13 +59,6 @@ def setUp(self): super().setUp() self.tmpdir = tempfile.mkdtemp() - def purge_environment(self): - """Remove any leftover easybuild variables""" - for var in os.environ.keys(): - # retain $EASYBUILD_IGNORECONFIGFILES, to make sure the test is isolated from system-wide config files! - if var.startswith('EASYBUILD_') and var != 'EASYBUILD_IGNORECONFIGFILES': - del os.environ[var] - def tearDown(self): """Clean up after a config test.""" super().tearDown() diff --git a/test/framework/options.py b/test/framework/options.py index af3835efe8..321062231b 100644 --- a/test/framework/options.py +++ b/test/framework/options.py @@ -120,13 +120,6 @@ def tearDown(self): super().tearDown() - def purge_environment(self): - """Remove any leftover easybuild variables""" - for var in os.environ.keys(): - # retain $EASYBUILD_IGNORECONFIGFILES, to make sure the test is isolated from system-wide config files! - if var.startswith('EASYBUILD_') and var != 'EASYBUILD_IGNORECONFIGFILES': - del os.environ[var] - def test_help_short(self, txt=None): """Test short help message.""" @@ -5249,7 +5242,7 @@ def test_show_config(self): 'EASYBUILD_SOURCEPATH', 'EASYBUILD_SOURCEPATH_DATA', ] - for key in os.environ.keys(): + for key in list(os.environ): if key.startswith('EASYBUILD_') and key not in retained_eb_env_vars: del os.environ[key] diff --git a/test/framework/style.py b/test/framework/style.py index 6305052619..92f588cf3b 100644 --- a/test/framework/style.py +++ b/test/framework/style.py @@ -64,7 +64,7 @@ def test_style_conformance(self): def test_check_trailing_whitespace(self): """Test for trailing whitespace check.""" if 'pycodestyle' not in sys.modules: - print("Skipping test_check_trailing_whitespace is not available") + print("Skipping test_check_trailing_whitespace pycodestyle is not available") return lines = [ diff --git a/test/framework/utilities.py b/test/framework/utilities.py index 66e1f4dc73..97dbc4cd9c 100644 --- a/test/framework/utilities.py +++ b/test/framework/utilities.py @@ -61,19 +61,22 @@ # involves ignoring any existing configuration files that are picked up, and cleaning the environment # this is tackled here rather than in suite.py, to make sure this is also done when test modules are ran separately -# clean up environment from unwanted $EASYBUILD_X env vars -for key in os.environ.keys(): - if key.startswith('%s_' % CONFIG_ENV_VAR_PREFIX): - del os.environ[key] +def remove_easybuild_environment_config_variables(exception=None): + """clean up environment from unwanted $EASYBUILD_X env vars""" + for key in list(os.environ): + if key.startswith(f'{CONFIG_ENV_VAR_PREFIX}_') and key != exception: + del os.environ[key] + # Ignore cmdline args as those are meant for the unittest framework # ignore any existing configuration files +remove_easybuild_environment_config_variables() go = EasyBuildOptions(go_args=[], go_useconfigfiles=False) os.environ['EASYBUILD_IGNORECONFIGFILES'] = ','.join(go.options.configfiles) # redefine $TEST_EASYBUILD_X env vars as $EASYBUILD_X test_env_var_prefix = 'TEST_EASYBUILD_' -for key in os.environ.keys(): +for key in list(os.environ): if key.startswith(test_env_var_prefix): val = os.environ[key] del os.environ[key] @@ -84,6 +87,11 @@ class EnhancedTestCase(TestCase): """Enhanced test case, provides extra functionality (e.g. an assertErrorRegex method).""" + def purge_environment(self): + """Remove any leftover easybuild variables""" + # retain $EASYBUILD_IGNORECONFIGFILES, to make sure the test is isolated from system-wide config files! + remove_easybuild_environment_config_variables(exception='EASYBUILD_IGNORECONFIGFILES') + def setUp(self): """Set up testcase.""" super().setUp()