use -nomp compiler option when OpenMP toolchain option is not enabled when using NVIDIA compilers (+ sync with develop)

.github/workflows/linting.yml

@@ -29,7 +29,7 @@ jobs:
     - name: install Python packages
       run: |
         pip install --upgrade pip
-        pip install --upgrade flake8
+        pip install --upgrade flake8 flake8-comprehensions
 
     - name: Run flake8 to verify PEP8-compliance of Python code
       run: |

.github/workflows/unit_tests.yml

@@ -9,6 +9,9 @@ concurrency:
   group: ${{format('{0}:{1}:{2}', github.repository, github.ref, github.workflow)}}
   cancel-in-progress: true
 
+env:
+  PYENV_ROOT: /opt/pyenv
+
 jobs:
   setup:
     runs-on: ubuntu-latest
@@ -22,6 +25,8 @@ jobs:
   build:
     needs: setup
     runs-on: ${{matrix.os || 'ubuntu-24.04'}}
+    env: 
+      JOB_OS: ${{matrix.os || 'ubuntu-24.04'}}
     strategy:
       matrix:
         # Python 3.10 is default in Ubuntu 22.04
@@ -34,6 +39,9 @@ jobs:
           - ${{needs.setup.outputs.modules5}}
         include:
           # Test different Python 3 versions with Lmod 8.x (with both Lua and Tcl module syntax)
+          - python: '3.6.1'
+            modules_tool: ${{needs.setup.outputs.lmod8}}
+            use_pyenv: '2.6.15'
           - python: '3.7'
             modules_tool: ${{needs.setup.outputs.lmod8}}
             os: ubuntu-22.04
@@ -56,13 +64,7 @@ jobs:
     steps:
     - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # 4.2.2
 
-    - name: set up Python
-      uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
-      with:
-        python-version: ${{matrix.python}}
-        architecture: x64
-
-    - name: install OS & Python packages
+    - name: install OS packages
       run: |
         # for modules tool
         APT_PKGS="lua5.3 liblua5.3-dev lua-filesystem lua-posix tcl tcl-dev"
@@ -71,6 +73,14 @@ jobs:
         # dep for GC3Pie
         APT_PKGS+=" time"
 
+        if [[ -n "${{matrix.use_pyenv}}" ]]; then
+          # See https://github.com/pyenv/pyenv/wiki#suggested-build-environment
+          APT_PKGS+=" libssl-dev zlib1g-dev \
+                    libbz2-dev libreadline-dev libsqlite3-dev curl git \
+                    libncursesw5-dev xz-utils tk-dev libxml2-dev libxmlsec1-dev libffi-dev liblzma-dev"
+          # Avoid segfault with older Pythons: https://github.com/pyenv/pyenv/issues/2239#issuecomment-1079275184
+          APT_PKGS+=" clang"
+        fi
         # Avoid apt-get update, as we don't really need it,
         # and it does more harm than good (it's fairly expensive, and it results in flaky test runs)
         if ! sudo apt-get install $APT_PKGS; then
@@ -79,7 +89,40 @@ jobs:
           sudo apt-get install $APT_PKGS
         fi
 
-        # Python packages
+    - name: set up Python
+      if: '!matrix.use_pyenv'
+      uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
+      with:
+        python-version: ${{matrix.python}}
+        architecture: x64
+
+    - name: Cache PyEnv
+      id: cache-pyenv
+      if: matrix.use_pyenv
+      uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # 4.3.0
+      with:
+        path: ${{env.PYENV_ROOT}}
+        key: ${{env.JOB_OS}}-py${{matrix.python}}-pyenv${{matrix.use_pyenv}}
+
+    - name: Setup PyEnv
+      if: matrix.use_pyenv
+      run: |
+        if [[ -z "${{steps.cache-pyenv.outputs.cache-hit}}" ]]; then
+          mkdir "$PYENV_ROOT"
+          wget https://github.com/pyenv/pyenv/archive/refs/tags/v${{matrix.use_pyenv}}.tar.gz -O- | tar xzf - -C "$PYENV_ROOT" --strip-components=1
+        fi
+        echo "${PYENV_ROOT}/bin" >> $GITHUB_PATH
+        export PATH="$PYENV_ROOT/bin:$PATH"
+        echo 'eval "$(pyenv init -)"' > ~/init_pyenv
+        . ~/init_pyenv
+        CC=clang pyenv install --skip-existing ${{matrix.python}}
+        echo 'pyenv global ${{matrix.python}}' >> ~/init_pyenv
+
+    - name: install Python packages
+      run: |
+        touch ~/init_pyenv # In case it doesn't exist, make empty file
+        . ~/init_pyenv
+        python -V
         pip --version
         pip install --upgrade pip
         pip --version
@@ -100,6 +143,7 @@ jobs:
         # and are only run after the PR gets merged
         GITHUB_TOKEN: ${{secrets.CI_UNIT_TESTS_GITHUB_TOKEN}}
       run: |
+        . ~/init_pyenv
         # only install GitHub token when testing with Lmod 8.x + Python 3.9, to avoid hitting GitHub rate limit
         # tests that require a GitHub token are skipped automatically when no GitHub token is available
         if [[ "${{matrix.modules_tool}}" =~ 'Lmod-8' ]] && [[ "${{matrix.python}}" =~ 3.9 ]]; then
@@ -126,13 +170,15 @@ jobs:
 
     - name: check sources
       run: |
+          . ~/init_pyenv
           # make sure there are no (top-level) "import setuptools" or "import pkg_resources" statements,
           # since EasyBuild should not have a runtime requirement on setuptools
           SETUPTOOLS_IMPORTS=$(egrep --exclude setup.py -RI '^(from|import)[ ]*pkg_resources|^(from|import)[ ]*setuptools' * || true)
           test "x$SETUPTOOLS_IMPORTS" = "x" || (echo "Found setuptools and/or pkg_resources imports in easybuild/:\n${SETUPTOOLS_IMPORTS}" && exit 1)
 
     - name: install sources
       run: |
+          . ~/init_pyenv
           # install from source distribution tarball, to test release as published on PyPI
           python setup.py sdist
           ls dist
@@ -142,7 +188,7 @@ jobs:
     - name: run test suite
       env:
         EB_VERBOSE: 1
-        LC_ALL: "" 
+        LC_ALL: ""
       run: |
           # run tests *outside* of checked out easybuild-framework directory,
           # to ensure we're testing installed version (see previous step)
@@ -158,7 +204,13 @@ jobs:
           export PATH=$PREFIX/bin:$(cat $HOME/path)
           export PYTHONPATH=$PREFIX/lib/python$(echo ${{matrix.python}} | cut -f1,2 -d'.')/site-packages:$PYTHONPATH
           echo "PYTHONPATH=$PYTHONPATH"
-          ls $(echo $PYTHONPATH | cut -f1:)
+          ls $(echo $PYTHONPATH | cut -f1 -d':')
+
+          # Do AFTER setting $PATH above
+          . ~/init_pyenv
+          which python3
+          python3 --version
+
           python3 -m easybuild.main --version
           eb --version
           # tell EasyBuild which modules tool is available

easybuild/base/exceptions.py

@@ -91,6 +91,7 @@ def __init__(self, msg, *args, **kwargs):
         if args:
             msg = msg % args
 
+        backtrace = []
         if self.LOC_INFO_TOP_PKG_NAMES is not None:
             # determine correct frame to fetch location information from
             frames_up = 1
@@ -101,19 +102,30 @@ def __init__(self, msg, *args, **kwargs):
             if self.INCLUDE_LOCATION:
                 # figure out where error was raised from
                 # current frame: this constructor, one frame above: location where LoggedException was created/raised
-                frameinfo = inspect.getouterframes(inspect.currentframe())[frames_up]
-
-                # determine short location of Python module where error was raised from,
-                # i.e. starting with an entry from LOC_INFO_TOP_PKG_NAMES
-                path_parts = frameinfo[1].split(os.path.sep)
-                if path_parts[0] == '':
-                    path_parts[0] = os.path.sep
-                top_indices = [path_parts.index(n) for n in self.LOC_INFO_TOP_PKG_NAMES if n in path_parts]
-                relpath = os.path.join(*path_parts[max(top_indices or [0]):])
+                frames = inspect.getouterframes(inspect.currentframe())[frames_up:]
+                backtrace = []
+                for frame in frames:
+                    # determine short location of Python module where error was raised from,
+                    # i.e. starting with an entry from LOC_INFO_TOP_PKG_NAMES
+                    path_parts = frame[1].split(os.path.sep)
+                    if path_parts[0] == '':
+                        path_parts[0] = os.path.sep
+                    try:
+                        top_idx = max(path_parts.index(n) for n in self.LOC_INFO_TOP_PKG_NAMES if n in path_parts)
+                    except ValueError:
+                        # If none found (outside PKG) stop backtrace if we have at least 1 entry
+                        if backtrace:
+                            break
+                        relpath = frame[1]
+                    else:
+                        relpath = os.path.join(*path_parts[top_idx:])
+                    backtrace.append(f'{relpath}:{frame[2]} in {frame[3]}')
 
                 # include location info at the end of the message
                 # for example: "Nope, giving up (at easybuild/tools/somemodule.py:123 in some_function)"
-                msg = "%s (at %s:%s in %s)" % (msg, relpath, frameinfo[2], frameinfo[3])
+                msg = f"{msg} (at {backtrace[0]})"
+
+        super().__init__(msg)
 
         logger = kwargs.get('logger', None)
         # try to use logger defined in caller's environment
@@ -123,6 +135,6 @@ def __init__(self, msg, *args, **kwargs):
             if logger is None:
                 logger = self.LOGGER_MODULE.getLogger()
 
+        if backtrace:
+            msg += '\nCallstack:\n\t' + '\n\t'.join(backtrace)
         getattr(logger, self.LOGGING_METHOD_NAME)(msg)
-
-        super().__init__(msg)

easybuild/base/fancylogger.py

@@ -435,7 +435,7 @@ def thread_name():
     """
     returns the current threads name
     """
-    return threading.currentThread().getName()
+    return threading.current_thread().name
 
 
 def getLogger(name=None, fname=False, clsname=False, fancyrecord=None):
@@ -577,8 +577,8 @@ def logToFile(filename, enable=True, filehandler=None, name=None, max_bytes=MAX_
         'mode': 'a',
         'maxBytes': max_bytes,
         'backupCount': backup_count,
+        'encoding': 'utf-8',
     }
-    handleropts['encoding'] = 'utf-8'
     # logging to a file is going to create the file later on, so let's try to be helpful and create the path if needed
     directory = os.path.dirname(filename)
     if not os.path.exists(directory):
@@ -783,7 +783,7 @@ def getAllExistingLoggers():
     """
     # not-so-well documented manager (in 2.6 and later)
     # return list of (name,logger) tuple
-    return [x for x in logging.Logger.manager.loggerDict.items()] + [(logging.root.name, logging.root)]
+    return list(logging.Logger.manager.loggerDict.items()) + [(logging.root.name, logging.root)]
 
 
 def getAllNonFancyloggers():

easybuild/base/generaloption.py

@@ -185,6 +185,8 @@ class ExtOption(CompleterOption):
            - set default to None if no option passed,
            - set to default if option without value passed,
            - set to value if option with value passed
+         - store_or_False: Same as store_or_None, but set to False instead of None
+           - Additionally supports --disable-
 
         Types:
           - strlist, strtuple : convert comma-separated string in a list resp. tuple of strings
@@ -196,14 +198,15 @@ class ExtOption(CompleterOption):
 
     ENABLE = 'enable'  # do nothing
     DISABLE = 'disable'  # inverse action
+    STORE_OR_FALSE = 'store_or_False'
 
     EXTOPTION_EXTRA_OPTIONS = ('date', 'datetime', 'regex', 'add', 'add_first', 'add_flex',)
-    EXTOPTION_STORE_OR = ('store_or_None', 'help')  # callback type
+    EXTOPTION_STORE_OR = ('store_or_None', STORE_OR_FALSE, 'help')  # callback type
     EXTOPTION_LOG = ('store_debuglog', 'store_infolog', 'store_warninglog',)
     EXTOPTION_HELP = ('shorthelp', 'confighelp', 'help')
 
     ACTIONS = Option.ACTIONS + EXTOPTION_EXTRA_OPTIONS + EXTOPTION_STORE_OR + EXTOPTION_LOG + EXTOPTION_HELP
-    STORE_ACTIONS = Option.STORE_ACTIONS + EXTOPTION_EXTRA_OPTIONS + EXTOPTION_LOG + ('store_or_None',)
+    STORE_ACTIONS = Option.STORE_ACTIONS + EXTOPTION_EXTRA_OPTIONS + EXTOPTION_LOG + ('store_or_None', STORE_OR_FALSE)
     TYPED_ACTIONS = Option.TYPED_ACTIONS + EXTOPTION_EXTRA_OPTIONS + EXTOPTION_STORE_OR
     ALWAYS_TYPED_ACTIONS = Option.ALWAYS_TYPED_ACTIONS + EXTOPTION_EXTRA_OPTIONS
 
@@ -232,7 +235,9 @@ def store_or(option, opt_str, value, parser, *args, **kwargs):  # pylint: disabl
                 """Callback for supporting options with optional values."""
                 # see http://stackoverflow.com/questions/1229146/parsing-empty-options-in-python
                 # ugly code, optparse is crap
-                if parser.rargs and not parser.rargs[0].startswith('-'):
+                if option.store_or == self.STORE_OR_FALSE and opt_str.startswith("--%s-" % self.DISABLE):
+                    val = False
+                elif parser.rargs and not parser.rargs[0].startswith('-'):
                     val = option.check_value(opt_str, parser.rargs.pop(0))
                 else:
                     val = kwargs.get('orig_default', None)
@@ -250,11 +255,7 @@ def store_or(option, opt_str, value, parser, *args, **kwargs):  # pylint: disabl
                 'orig_default': copy.deepcopy(self.default),
             }
             self.action = 'callback'  # act as callback
-
-            if self.store_or in self.EXTOPTION_STORE_OR:
-                self.default = None
-            else:
-                self.log.raiseException("_set_attrs: unknown store_or %s" % self.store_or, exception=ValueError)
+            self.default = False if self.action == self.STORE_OR_FALSE else None
 
     def process(self, opt, value, values, parser):
         """Handle option-as-value issues before actually processing option."""
@@ -1219,6 +1220,8 @@ def add_group_parser(self, opt_dict, description, prefix=None, otherdefaults=Non
                 if action in self.parser.option_class.BOOLEAN_ACTIONS:
                     args.append("--%s-%s" % (self.parser.option_class.ENABLE, opt_name))
                     args.append("--%s-%s" % (self.parser.option_class.DISABLE, opt_name))
+                elif action == self.parser.option_class.STORE_OR_FALSE:
+                    args.append("--%s-%s" % (self.parser.option_class.DISABLE, opt_name))
 
             # force passed_kwargs as final nameds
             nameds.update(passed_kwargs)

easybuild/base/testing.py

@@ -201,6 +201,15 @@ def get_stderr(self):
         """Return output captured from stderr until now."""
         return sys.stderr.getvalue()
 
+    @contextmanager
+    def mocked_stdout(self):
+        """Context manager to mock stdout"""
+        self.mock_stdout(True)
+        try:
+            yield sys.stdout
+        finally:
+            self.mock_stdout(False)
+
     @contextmanager
     def mocked_stdout_stderr(self, mock_stdout=True, mock_stderr=True):
         """Context manager to mock stdout and stderr"""
@@ -221,6 +230,16 @@ def mocked_stdout_stderr(self, mock_stdout=True, mock_stderr=True):
             if mock_stderr:
                 self.mock_stderr(False)
 
+    @contextmanager
+    def saved_env(self):
+        """Context manager to reset environment to state when it was entered"""
+        orig_env = os.environ.copy()
+        try:
+            yield
+        finally:
+            os.environ.clear()
+            os.environ.update(orig_env)
+
     def tearDown(self):
         """Cleanup after running a test."""
         self.mock_stdout(False)

easybuild/base/wrapper.py

@@ -39,7 +39,7 @@ def proxy(self, *args):  # pylint:disable=unused-argument
         # create proxies for wrapped object's double-underscore attributes
         type.__init__(cls, name, bases, dct)
         if cls.__wraps__:
-            ignore = set("__%s__" % n for n in cls.__ignore__.split())
+            ignore = {"__%s__" % n for n in cls.__ignore__.split()}
             for name in dir(cls.__wraps__):
                 if name.startswith("__"):
                     if name not in ignore and name not in dct: