Add back CI for Python 3.6 by using pyenv

.github/workflows/unit_tests.yml

@@ -9,6 +9,9 @@ concurrency:
   group: ${{format('{0}:{1}:{2}', github.repository, github.ref, github.workflow)}}
   cancel-in-progress: true
 
+env:
+  PYENV_ROOT: /opt/pyenv
+
 jobs:
   setup:
     runs-on: ubuntu-latest
@@ -22,6 +25,8 @@ jobs:
   build:
     needs: setup
     runs-on: ${{matrix.os || 'ubuntu-24.04'}}
+    env: 
+      JOB_OS: ${{matrix.os || 'ubuntu-24.04'}}
     strategy:
       matrix:
         # Python 3.10 is default in Ubuntu 22.04
@@ -34,6 +39,9 @@ jobs:
           - ${{needs.setup.outputs.modules5}}
         include:
           # Test different Python 3 versions with Lmod 8.x (with both Lua and Tcl module syntax)
+          - python: '3.6.1'
+            modules_tool: ${{needs.setup.outputs.lmod8}}
+            use_pyenv: '2.6.15'
           - python: '3.7'
             modules_tool: ${{needs.setup.outputs.lmod8}}
             os: ubuntu-22.04
@@ -56,13 +64,7 @@ jobs:
     steps:
     - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # 4.2.2
 
-    - name: set up Python
-      uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
-      with:
-        python-version: ${{matrix.python}}
-        architecture: x64
-
-    - name: install OS & Python packages
+    - name: install OS packages
       run: |
         # for modules tool
         APT_PKGS="lua5.3 liblua5.3-dev lua-filesystem lua-posix tcl tcl-dev"
@@ -71,6 +73,14 @@ jobs:
         # dep for GC3Pie
         APT_PKGS+=" time"
 
+        if [[ -n "${{matrix.use_pyenv}}" ]]; then
+          # See https://github.com/pyenv/pyenv/wiki#suggested-build-environment
+          APT_PKGS+=" libssl-dev zlib1g-dev \
+                    libbz2-dev libreadline-dev libsqlite3-dev curl git \
+                    libncursesw5-dev xz-utils tk-dev libxml2-dev libxmlsec1-dev libffi-dev liblzma-dev"
+          # Avoid segfault with older Pythons: https://github.com/pyenv/pyenv/issues/2239#issuecomment-1079275184
+          APT_PKGS+=" clang"
+        fi
         # Avoid apt-get update, as we don't really need it,
         # and it does more harm than good (it's fairly expensive, and it results in flaky test runs)
         if ! sudo apt-get install $APT_PKGS; then
@@ -79,7 +89,40 @@ jobs:
           sudo apt-get install $APT_PKGS
         fi
 
-        # Python packages
+    - name: set up Python
+      if: '!matrix.use_pyenv'
+      uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
+      with:
+        python-version: ${{matrix.python}}
+        architecture: x64
+
+    - name: Cache PyEnv
+      id: cache-pyenv
+      if: matrix.use_pyenv
+      uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # 4.3.0
+      with:
+        path: ${{env.PYENV_ROOT}}
+        key: ${{env.JOB_OS}}-py${{matrix.python}}-pyenv${{matrix.use_pyenv}}
+
+    - name: Setup PyEnv
+      if: matrix.use_pyenv
+      run: |
+        if [[ -z "${{steps.cache-pyenv.outputs.cache-hit}}" ]]; then
+          mkdir "$PYENV_ROOT"
+          wget https://github.com/pyenv/pyenv/archive/refs/tags/v${{matrix.use_pyenv}}.tar.gz -O- | tar xzf - -C "$PYENV_ROOT" --strip-components=1
+        fi
+        echo "${PYENV_ROOT}/bin" >> $GITHUB_PATH
+        export PATH="$PYENV_ROOT/bin:$PATH"
+        echo 'eval "$(pyenv init -)"' > ~/init_pyenv
+        . ~/init_pyenv
+        CC=clang pyenv install --skip-existing ${{matrix.python}}
+        echo 'pyenv global ${{matrix.python}}' >> ~/init_pyenv
+
+    - name: install Python packages
+      run: |
+        touch ~/init_pyenv # In case it doesn't exist, make empty file
+        . ~/init_pyenv
+        python -V
         pip --version
         pip install --upgrade pip
         pip --version
@@ -100,6 +143,7 @@ jobs:
         # and are only run after the PR gets merged
         GITHUB_TOKEN: ${{secrets.CI_UNIT_TESTS_GITHUB_TOKEN}}
       run: |
+        . ~/init_pyenv
         # only install GitHub token when testing with Lmod 8.x + Python 3.9, to avoid hitting GitHub rate limit
         # tests that require a GitHub token are skipped automatically when no GitHub token is available
         if [[ "${{matrix.modules_tool}}" =~ 'Lmod-8' ]] && [[ "${{matrix.python}}" =~ 3.9 ]]; then
@@ -126,13 +170,15 @@ jobs:
 
     - name: check sources
       run: |
+          . ~/init_pyenv
           # make sure there are no (top-level) "import setuptools" or "import pkg_resources" statements,
           # since EasyBuild should not have a runtime requirement on setuptools
           SETUPTOOLS_IMPORTS=$(egrep --exclude setup.py -RI '^(from|import)[ ]*pkg_resources|^(from|import)[ ]*setuptools' * || true)
           test "x$SETUPTOOLS_IMPORTS" = "x" || (echo "Found setuptools and/or pkg_resources imports in easybuild/:\n${SETUPTOOLS_IMPORTS}" && exit 1)
 
     - name: install sources
       run: |
+          . ~/init_pyenv
           # install from source distribution tarball, to test release as published on PyPI
           python setup.py sdist
           ls dist
@@ -142,7 +188,7 @@ jobs:
     - name: run test suite
       env:
         EB_VERBOSE: 1
-        LC_ALL: "" 
+        LC_ALL: ""
       run: |
           # run tests *outside* of checked out easybuild-framework directory,
           # to ensure we're testing installed version (see previous step)
@@ -158,7 +204,13 @@ jobs:
           export PATH=$PREFIX/bin:$(cat $HOME/path)
           export PYTHONPATH=$PREFIX/lib/python$(echo ${{matrix.python}} | cut -f1,2 -d'.')/site-packages:$PYTHONPATH
           echo "PYTHONPATH=$PYTHONPATH"
-          ls $(echo $PYTHONPATH | cut -f1:)
+          ls $(echo $PYTHONPATH | cut -f1 -d':')
+
+          # Do AFTER setting $PATH above
+          . ~/init_pyenv
+          which python3
+          python3 --version
+
           python3 -m easybuild.main --version
           eb --version
           # tell EasyBuild which modules tool is available

easybuild/tools/filetools.py

@@ -2632,6 +2632,16 @@ def copy_files(paths, target_path, force_in_dry_run=False, target_single_file=Fa
         raise EasyBuildError("One or more files to copy should be specified!")
 
 
+def is_recursive_symlink(path) -> bool:
+    """Check if the given path is a symlink and points to itself"""
+    if not os.path.islink(path):
+        return False
+    abs_path = os.path.abspath(path)
+    linkpath = os.path.realpath(abs_path)
+    abs_path += os.sep  # To catch the case where both are equal
+    return abs_path.startswith(linkpath + os.sep)
+
+
 def has_recursive_symlinks(path):
     """
     Check the given directory for recursive symlinks.
@@ -2643,12 +2653,9 @@ def has_recursive_symlinks(path):
     for dirpath, dirnames, filenames in os.walk(path, followlinks=True):
         for name in itertools.chain(dirnames, filenames):
             fullpath = os.path.join(dirpath, name)
-            if os.path.islink(fullpath):
-                linkpath = os.path.realpath(fullpath)
-                fullpath += os.sep  # To catch the case where both are equal
-                if fullpath.startswith(linkpath + os.sep):
-                    _log.info("Recursive symlink detected at %s", fullpath)
-                    return True
+            if is_recursive_symlink(fullpath):
+                _log.info("Recursive symlink detected at %s", fullpath)
+                return True
     return False
 
 
@@ -2713,7 +2720,20 @@ def copy_dir(path, target_path, force_in_dry_run=False, dirs_exist_ok=False, che
 
             else:
                 # if dirs_exist_ok is not enabled or target directory doesn't exist, just use shutil.copytree
-                shutil.copytree(path, target_path, **kwargs)
+                if sys.version_info < (3, 7) and kwargs.get('symlinks'):
+                    # Python 3.6 might not correctly detect support for chmod on broken symlinks
+                    # This was fixed in 3.7.0
+                    # Approach taken from https://bugs.python.org/issue6547: Fail only if all errors are due to symlinks
+                    try:
+                        shutil.copytree(path, target_path, **kwargs)
+                    except shutil.Error as e:
+                        if all(os.path.islink(src) and not os.path.exists(src) or is_recursive_symlink(src)
+                               for src, _dst, _err in e.args[0]):
+                            _log.info("Ignoring errors when copying broken symlinks: %s", e.args[0])
+                        else:
+                            raise
+                else:
+                    shutil.copytree(path, target_path, **kwargs)
 
             _log.info("%s copied to %s", path, target_path)
         except (IOError, OSError, shutil.Error) as err:

requirements.txt

@@ -17,6 +17,12 @@ python-graph-dot
 python-hglib
 requests
 
-archspec
-
+# Some packages dropped support for Python 3.6, so specify max versions
+rich<13; python_version < '3.7'
 rich
+
+# Similar for dependencies of other packages
+cryptography<37; python_version < '3.7'
+PyNaCl<1.5; python_version < '3.7'
+
+archspec

test/framework/config.py

@@ -59,13 +59,6 @@ def setUp(self):
         super().setUp()
         self.tmpdir = tempfile.mkdtemp()
 
-    def purge_environment(self):
-        """Remove any leftover easybuild variables"""
-        for var in os.environ.keys():
-            # retain $EASYBUILD_IGNORECONFIGFILES, to make sure the test is isolated from system-wide config files!
-            if var.startswith('EASYBUILD_') and var != 'EASYBUILD_IGNORECONFIGFILES':
-                del os.environ[var]
-
     def tearDown(self):
         """Clean up after a config test."""
         super().tearDown()

test/framework/options.py

@@ -120,13 +120,6 @@ def tearDown(self):
 
         super().tearDown()
 
-    def purge_environment(self):
-        """Remove any leftover easybuild variables"""
-        for var in os.environ.keys():
-            # retain $EASYBUILD_IGNORECONFIGFILES, to make sure the test is isolated from system-wide config files!
-            if var.startswith('EASYBUILD_') and var != 'EASYBUILD_IGNORECONFIGFILES':
-                del os.environ[var]
-
     def test_help_short(self, txt=None):
         """Test short help message."""
 
@@ -5249,7 +5242,7 @@ def test_show_config(self):
             'EASYBUILD_SOURCEPATH',
             'EASYBUILD_SOURCEPATH_DATA',
         ]
-        for key in os.environ.keys():
+        for key in list(os.environ):
             if key.startswith('EASYBUILD_') and key not in retained_eb_env_vars:
                 del os.environ[key]
 

test/framework/style.py

@@ -64,7 +64,7 @@ def test_style_conformance(self):
     def test_check_trailing_whitespace(self):
         """Test for trailing whitespace check."""
         if 'pycodestyle' not in sys.modules:
-            print("Skipping test_check_trailing_whitespace is not available")
+            print("Skipping test_check_trailing_whitespace pycodestyle is not available")
             return
 
         lines = [

test/framework/utilities.py

@@ -61,19 +61,22 @@
 # involves ignoring any existing configuration files that are picked up, and cleaning the environment
 # this is tackled here rather than in suite.py, to make sure this is also done when test modules are ran separately
 
-# clean up environment from unwanted $EASYBUILD_X env vars
-for key in os.environ.keys():
-    if key.startswith('%s_' % CONFIG_ENV_VAR_PREFIX):
-        del os.environ[key]
+def remove_easybuild_environment_config_variables(exception=None):
+    """clean up environment from unwanted $EASYBUILD_X env vars"""
+    for key in list(os.environ):
+        if key.startswith(f'{CONFIG_ENV_VAR_PREFIX}_') and key != exception:
+            del os.environ[key]
+
 
 # Ignore cmdline args as those are meant for the unittest framework
 # ignore any existing configuration files
+remove_easybuild_environment_config_variables()
 go = EasyBuildOptions(go_args=[], go_useconfigfiles=False)
 os.environ['EASYBUILD_IGNORECONFIGFILES'] = ','.join(go.options.configfiles)
 
 # redefine $TEST_EASYBUILD_X env vars as $EASYBUILD_X
 test_env_var_prefix = 'TEST_EASYBUILD_'
-for key in os.environ.keys():
+for key in list(os.environ):
     if key.startswith(test_env_var_prefix):
         val = os.environ[key]
         del os.environ[key]
@@ -84,6 +87,11 @@
 class EnhancedTestCase(TestCase):
     """Enhanced test case, provides extra functionality (e.g. an assertErrorRegex method)."""
 
+    def purge_environment(self):
+        """Remove any leftover easybuild variables"""
+        # retain $EASYBUILD_IGNORECONFIGFILES, to make sure the test is isolated from system-wide config files!
+        remove_easybuild_environment_config_variables(exception='EASYBUILD_IGNORECONFIGFILES')
+
     def setUp(self):
         """Set up testcase."""
         super().setUp()