From 27c32ef91b174bf31777c9a4e6c57eebbbd0392d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Mikael=20=C3=96hman?= <micketeer@gmail.com>
Date: Wed, 22 May 2024 19:12:03 +0200
Subject: [PATCH 01/25] Add option for preferring EBPYTHONPREFIXES

---
 easybuild/framework/easyconfig/default.py | 1 +
 easybuild/tools/config.py                 | 1 +
 easybuild/tools/options.py                | 2 ++
 3 files changed, 4 insertions(+)

diff --git a/easybuild/framework/easyconfig/default.py b/easybuild/framework/easyconfig/default.py
index 438bcea4e1..43f6093588 100644
--- a/easybuild/framework/easyconfig/default.py
+++ b/easybuild/framework/easyconfig/default.py
@@ -115,6 +115,7 @@
     'patches': [[], "List of patches to apply", BUILD],
     'prebuildopts': ['', 'Extra options pre-passed to build command.', BUILD],
     'preconfigopts': ['', 'Extra options pre-passed to configure.', BUILD],
+    'prefer_ebpythonprefixes': [True, "Prefer EBPYTHONPREFIXES over PYTHONPATH when possible.", BUILD],
     'preinstallopts': ['', 'Extra prefix options for installation.', BUILD],
     'pretestopts': ['', 'Extra prefix options for test.', BUILD],
     'postinstallcmds': [[], 'Commands to run after the install step.', BUILD],
diff --git a/easybuild/tools/config.py b/easybuild/tools/config.py
index 13f9722bce..67473b4dc0 100644
--- a/easybuild/tools/config.py
+++ b/easybuild/tools/config.py
@@ -336,6 +336,7 @@ def mk_full_default_path(name, prefix=DEFAULT_PREFIX):
         'modules_tool_version_check',
         'mpi_tests',
         'pre_create_installdir',
+        'prefer_ebpythonprefixes',
         'show_progress_bar',
         'trace',
     ],
diff --git a/easybuild/tools/options.py b/easybuild/tools/options.py
index a61daa6d1c..c5fc72061d 100644
--- a/easybuild/tools/options.py
+++ b/easybuild/tools/options.py
@@ -490,6 +490,8 @@ def override_options(self):
                                             None, 'store_true', False),
             'pre-create-installdir': ("Create installation directory before submitting build jobs",
                                       None, 'store_true', True),
+            'prefer-ebpythonprefixes': (("Prefer EBPYTHONPREFIXES over PYTHONPATH when possible"),
+                                        None, 'store_true', True),
             'pretend': (("Does the build/installation in a test directory located in $HOME/easybuildinstall"),
                         None, 'store_true', False, 'p'),
             'read-only-installdir': ("Set read-only permissions on installation directory after installation",

From 46525093d7b622534f31b45ac45bbac1b261d2b6 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Mikael=20=C3=96hman?= <micketeer@gmail.com>
Date: Wed, 22 May 2024 19:55:17 +0200
Subject: [PATCH 02/25] Add automatic detection of python site-package dir in
 installations

---
 easybuild/framework/easyblock.py | 16 ++++++++++++++++
 1 file changed, 16 insertions(+)

diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py
index 88da255d1d..6e7ab1d654 100644
--- a/easybuild/framework/easyblock.py
+++ b/easybuild/framework/easyblock.py
@@ -1438,6 +1438,22 @@ def make_module_extra(self, altroot=None, altversion=None):
                                      value, type(value))
             lines.append(self.module_generator.append_paths(key, value, allow_abs=self.cfg['allow_append_abs_path']))
 
+        # Add automatic PYTHONPATH or EBPYTHONPREFIXES if they aren't already present
+        if 'PYTHONPATH' not in self.module_generator.added_paths_per_key and \
+           'EBPYTHONPREFIXES' not in self.module_generator.added_paths_per_key:
+            python_paths = [path for path in glob.glob('lib*/python*/site-packages')
+                            if re.match(r'lib(64)?/python\d+\.\d+/site-packages', path)]
+            use_ebpythonprefixes = get_software_root('Python') and build_option('prefer_ebpythonprefixes') and \
+                self.cfg['prefer_ebpythonprefixes']
+
+            if len(python_paths) > 1 and not use_ebpythonprefixes:
+                raise EasyBuildError('Multiple python paths requires EBPYHONPREFIXES: ' + ', '.join(python_paths))
+            elif python_paths:
+                if use_ebpythonprefixes:
+                    lines.append(self.module_generator.append_paths('EBPYHONPREFIXES', '.'))
+                else:
+                    lines.append(self.module_generator.append_paths('PYTHONPATH', python_paths))
+
         modloadmsg = self.cfg['modloadmsg']
         if modloadmsg:
             # add trailing newline to prevent that shell prompt is 'glued' to module load message

From 1acb45a1d39abf4f889dc193edac28cf2b2155de Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Mikael=20=C3=96hman?= <micketeer@gmail.com>
Date: Wed, 3 Jul 2024 19:54:49 +0200
Subject: [PATCH 03/25] Switch to using prepend_paths instead of append_paths

---
 easybuild/framework/easyblock.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py
index 6e7ab1d654..a4cf035958 100644
--- a/easybuild/framework/easyblock.py
+++ b/easybuild/framework/easyblock.py
@@ -1447,12 +1447,12 @@ def make_module_extra(self, altroot=None, altversion=None):
                 self.cfg['prefer_ebpythonprefixes']
 
             if len(python_paths) > 1 and not use_ebpythonprefixes:
-                raise EasyBuildError('Multiple python paths requires EBPYHONPREFIXES: ' + ', '.join(python_paths))
+                raise EasyBuildError('Multiple python paths requires EBPYTHONPREFIXES: ' + ', '.join(python_paths))
             elif python_paths:
                 if use_ebpythonprefixes:
-                    lines.append(self.module_generator.append_paths('EBPYHONPREFIXES', '.'))
+                    lines.append(self.module_generator.prepend_paths('EBPYTHONPREFIXES', '.'))
                 else:
-                    lines.append(self.module_generator.append_paths('PYTHONPATH', python_paths))
+                    lines.append(self.module_generator.prepend_paths('PYTHONPATH', python_paths))
 
         modloadmsg = self.cfg['modloadmsg']
         if modloadmsg:

From d1c889f8fe0939a76aab3031af384baf8a8cacd6 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Mikael=20=C3=96hman?= <micketeer@gmail.com>
Date: Thu, 4 Jul 2024 02:34:05 +0200
Subject: [PATCH 04/25] Fix relative paths when globbing for python paths

---
 easybuild/framework/easyblock.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py
index a4cf035958..26fb7c9e5b 100644
--- a/easybuild/framework/easyblock.py
+++ b/easybuild/framework/easyblock.py
@@ -1441,8 +1441,9 @@ def make_module_extra(self, altroot=None, altversion=None):
         # Add automatic PYTHONPATH or EBPYTHONPREFIXES if they aren't already present
         if 'PYTHONPATH' not in self.module_generator.added_paths_per_key and \
            'EBPYTHONPREFIXES' not in self.module_generator.added_paths_per_key:
-            python_paths = [path for path in glob.glob('lib*/python*/site-packages')
-                            if re.match(r'lib(64)?/python\d+\.\d+/site-packages', path)]
+            python_paths = [path[len(self.installdir)+1:]
+                            for path in glob.glob(f'{self.installdir}/lib*/python*/site-packages')
+                            if re.match(self.installdir + r'/lib(64)?/python\d+\.\d+/site-packages', path)]
             use_ebpythonprefixes = get_software_root('Python') and build_option('prefer_ebpythonprefixes') and \
                 self.cfg['prefer_ebpythonprefixes']
 

From 0310bed9a1c9c21bb961dfe04ce00ab34e9a3b76 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Mikael=20=C3=96hman?= <micketeer@gmail.com>
Date: Thu, 4 Jul 2024 15:08:55 +0200
Subject: [PATCH 05/25] Simplify generated EBPYTHONPREFIXES path

---
 easybuild/framework/easyblock.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py
index 26fb7c9e5b..6cc01f7498 100644
--- a/easybuild/framework/easyblock.py
+++ b/easybuild/framework/easyblock.py
@@ -1451,7 +1451,7 @@ def make_module_extra(self, altroot=None, altversion=None):
                 raise EasyBuildError('Multiple python paths requires EBPYTHONPREFIXES: ' + ', '.join(python_paths))
             elif python_paths:
                 if use_ebpythonprefixes:
-                    lines.append(self.module_generator.prepend_paths('EBPYTHONPREFIXES', '.'))
+                    lines.append(self.module_generator.prepend_paths('EBPYTHONPREFIXES', ''))
                 else:
                     lines.append(self.module_generator.prepend_paths('PYTHONPATH', python_paths))
 

From a1e4ecced7e941aba8ecafe08421d27472dc654a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Mikael=20=C3=96hman?= <micketeer@gmail.com>
Date: Sat, 6 Jul 2024 20:07:18 +0200
Subject: [PATCH 06/25] Rework logic to allow nonstandard PYTHONPATHs

to be added while still preserving the automatic adding of standard
paths.
---
 easybuild/framework/easyblock.py | 29 ++++++++++++++---------------
 1 file changed, 14 insertions(+), 15 deletions(-)

diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py
index 6cc01f7498..66b090f915 100644
--- a/easybuild/framework/easyblock.py
+++ b/easybuild/framework/easyblock.py
@@ -1439,21 +1439,20 @@ def make_module_extra(self, altroot=None, altversion=None):
             lines.append(self.module_generator.append_paths(key, value, allow_abs=self.cfg['allow_append_abs_path']))
 
         # Add automatic PYTHONPATH or EBPYTHONPREFIXES if they aren't already present
-        if 'PYTHONPATH' not in self.module_generator.added_paths_per_key and \
-           'EBPYTHONPREFIXES' not in self.module_generator.added_paths_per_key:
-            python_paths = [path[len(self.installdir)+1:]
-                            for path in glob.glob(f'{self.installdir}/lib*/python*/site-packages')
-                            if re.match(self.installdir + r'/lib(64)?/python\d+\.\d+/site-packages', path)]
-            use_ebpythonprefixes = get_software_root('Python') and build_option('prefer_ebpythonprefixes') and \
-                self.cfg['prefer_ebpythonprefixes']
-
-            if len(python_paths) > 1 and not use_ebpythonprefixes:
-                raise EasyBuildError('Multiple python paths requires EBPYTHONPREFIXES: ' + ', '.join(python_paths))
-            elif python_paths:
-                if use_ebpythonprefixes:
-                    lines.append(self.module_generator.prepend_paths('EBPYTHONPREFIXES', ''))
-                else:
-                    lines.append(self.module_generator.prepend_paths('PYTHONPATH', python_paths))
+        python_paths = [path[len(self.installdir)+1:]
+                        for path in glob.glob(f'{self.installdir}/lib*/python*/site-packages')
+                        if re.match(self.installdir + r'/lib(64)?/python\d+\.\d+/site-packages', path)]
+        use_ebpythonprefixes = get_software_root('Python') and build_option('prefer_ebpythonprefixes') and \
+            self.cfg['prefer_ebpythonprefixes']
+
+        if len(python_paths) > 1 and not use_ebpythonprefixes:
+            raise EasyBuildError('Multiple python paths requires EBPYTHONPREFIXES: ' + ', '.join(python_paths))
+        elif python_paths:
+            # Add paths unless they were already added
+            if use_ebpythonprefixes and '' not in self.module_generator.added_paths_per_key['EBPYTHONPREFIXES']:
+                lines.append(self.module_generator.prepend_paths('EBPYTHONPREFIXES', ''))
+            elif python_paths[0] not in self.module_generator.added_paths_per_key['PYTHONPATH']:
+                lines.append(self.module_generator.prepend_paths('PYTHONPATH', python_paths))
 
         modloadmsg = self.cfg['modloadmsg']
         if modloadmsg:

From 4ed16be0294cd9efc9ea4e95154daf01d5108aa1 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Mikael=20=C3=96hman?= <micketeer@gmail.com>
Date: Sun, 7 Jul 2024 18:00:28 +0200
Subject: [PATCH 07/25] Use defaultdict with sets for added_paths_per_key

Simplifies code when using it in easyblocks
---
 easybuild/tools/module_generator.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/easybuild/tools/module_generator.py b/easybuild/tools/module_generator.py
index a05d9b8941..e9e285b5cb 100644
--- a/easybuild/tools/module_generator.py
+++ b/easybuild/tools/module_generator.py
@@ -39,6 +39,7 @@
 import os
 import re
 import tempfile
+from collections import defaultdict
 from contextlib import contextmanager
 from easybuild.tools import LooseVersion
 from textwrap import wrap
@@ -153,7 +154,7 @@ def start_module_creation(self):
             raise EasyBuildError('Module creation already in process. '
                                  'You cannot create multiple modules at the same time!')
         # Mapping of keys/env vars to paths already added
-        self.added_paths_per_key = dict()
+        self.added_paths_per_key = defaultdict(set)
         txt = self.MODULE_SHEBANG
         if txt:
             txt += '\n'
@@ -212,7 +213,7 @@ def _filter_paths(self, key, paths):
             print_warning('Module creation has not been started. Call start_module_creation first!')
             return paths
 
-        added_paths = self.added_paths_per_key.setdefault(key, set())
+        added_paths = self.added_paths_per_key[key]
         # paths can be a string
         if isinstance(paths, str):
             if paths in added_paths:

From 9f3cf6aeb9fe17b72123a9d99c3963ceee25e00a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Mikael=20=C3=96hman?= <micketeer@gmail.com>
Date: Tue, 13 Aug 2024 11:23:11 +0200
Subject: [PATCH 08/25] Exclude base python installations themselves from
 automatic PYTHONPATH

---
 easybuild/framework/easyblock.py | 39 ++++++++++++++++++++------------
 1 file changed, 24 insertions(+), 15 deletions(-)

diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py
index 66b090f915..898fb7b802 100644
--- a/easybuild/framework/easyblock.py
+++ b/easybuild/framework/easyblock.py
@@ -1438,21 +1438,30 @@ def make_module_extra(self, altroot=None, altversion=None):
                                      value, type(value))
             lines.append(self.module_generator.append_paths(key, value, allow_abs=self.cfg['allow_append_abs_path']))
 
-        # Add automatic PYTHONPATH or EBPYTHONPREFIXES if they aren't already present
-        python_paths = [path[len(self.installdir)+1:]
-                        for path in glob.glob(f'{self.installdir}/lib*/python*/site-packages')
-                        if re.match(self.installdir + r'/lib(64)?/python\d+\.\d+/site-packages', path)]
-        use_ebpythonprefixes = get_software_root('Python') and build_option('prefer_ebpythonprefixes') and \
-            self.cfg['prefer_ebpythonprefixes']
-
-        if len(python_paths) > 1 and not use_ebpythonprefixes:
-            raise EasyBuildError('Multiple python paths requires EBPYTHONPREFIXES: ' + ', '.join(python_paths))
-        elif python_paths:
-            # Add paths unless they were already added
-            if use_ebpythonprefixes and '' not in self.module_generator.added_paths_per_key['EBPYTHONPREFIXES']:
-                lines.append(self.module_generator.prepend_paths('EBPYTHONPREFIXES', ''))
-            elif python_paths[0] not in self.module_generator.added_paths_per_key['PYTHONPATH']:
-                lines.append(self.module_generator.prepend_paths('PYTHONPATH', python_paths))
+        # Add automatic PYTHONPATH or EBPYTHONPREFIXES if they aren't already present and python paths exist
+        if self.name not in ['Python', 'Miniconda', 'Anaconda']:  # nothing extra needed for base python installations
+            # Exclude symlinks lib -> lib64 or vice verse to avoid duplicates
+            python_paths = []
+            if not os.path.islink(f'{self.installdir}/lib'):
+                python_paths += [path[len(self.installdir)+1:]
+                                 for path in glob.glob(f'{self.installdir}/lib/python*/site-packages')
+                                 if re.match(self.installdir + r'/lib/python\d+\.\d+/site-packages', path)]
+            if not os.path.islink(f'{self.installdir}/lib64'):
+                python_paths += [path[len(self.installdir)+1:]
+                                 for path in glob.glob(f'{self.installdir}/lib64/python*/site-packages')
+                                 if re.match(self.installdir + r'/lib64/python\d+\.\d+/site-packages', path)]
+
+            use_ebpythonprefixes = get_software_root('Python') and build_option('prefer_ebpythonprefixes') and \
+                self.cfg['prefer_ebpythonprefixes']
+
+            if len(python_paths) > 1 and not use_ebpythonprefixes:
+                raise EasyBuildError('Multiple python paths requires EBPYTHONPREFIXES: ' + ', '.join(python_paths))
+            elif python_paths:
+                # Add paths unless they were already added
+                if use_ebpythonprefixes and '' not in self.module_generator.added_paths_per_key['EBPYTHONPREFIXES']:
+                    lines.append(self.module_generator.prepend_paths('EBPYTHONPREFIXES', ''))
+                elif python_paths[0] not in self.module_generator.added_paths_per_key['PYTHONPATH']:
+                    lines.append(self.module_generator.prepend_paths('PYTHONPATH', python_paths))
 
         modloadmsg = self.cfg['modloadmsg']
         if modloadmsg:

From 037827b6a47d38550e66ddb0b68d03a5b297c4a3 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Mikael=20=C3=96hman?= <micketeer@gmail.com>
Date: Thu, 15 Aug 2024 15:23:06 +0200
Subject: [PATCH 09/25] Fix logic when someone else added
 PYTHONPATH/EBPYTHONPREFIXES

---
 easybuild/framework/easyblock.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py
index 898fb7b802..0397d74b94 100644
--- a/easybuild/framework/easyblock.py
+++ b/easybuild/framework/easyblock.py
@@ -1458,8 +1458,9 @@ def make_module_extra(self, altroot=None, altversion=None):
                 raise EasyBuildError('Multiple python paths requires EBPYTHONPREFIXES: ' + ', '.join(python_paths))
             elif python_paths:
                 # Add paths unless they were already added
-                if use_ebpythonprefixes and '' not in self.module_generator.added_paths_per_key['EBPYTHONPREFIXES']:
-                    lines.append(self.module_generator.prepend_paths('EBPYTHONPREFIXES', ''))
+                if use_ebpythonprefixes:
+                    if '' not in self.module_generator.added_paths_per_key['EBPYTHONPREFIXES']:
+                        lines.append(self.module_generator.prepend_paths('EBPYTHONPREFIXES', ''))
                 elif python_paths[0] not in self.module_generator.added_paths_per_key['PYTHONPATH']:
                     lines.append(self.module_generator.prepend_paths('PYTHONPATH', python_paths))
 

From 38da36b85528b731b294d531732304d756780d32 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Mikael=20=C3=96hman?= <micketeer@gmail.com>
Date: Thu, 15 Aug 2024 20:26:57 +0200
Subject: [PATCH 10/25] Add runtime_only flag to dependencies()

---
 easybuild/framework/easyconfig/easyconfig.py | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/easybuild/framework/easyconfig/easyconfig.py b/easybuild/framework/easyconfig/easyconfig.py
index 0d54443517..4a4c4eabd6 100644
--- a/easybuild/framework/easyconfig/easyconfig.py
+++ b/easybuild/framework/easyconfig/easyconfig.py
@@ -1105,15 +1105,19 @@ def filter_deps(self, deps):
 
         return retained_deps
 
-    def dependencies(self, build_only=False):
+    def dependencies(self, build_only=False, runtime_only=False):
         """
         Returns an array of parsed dependencies (after filtering, if requested)
         dependency = {'name': '', 'version': '', 'system': (False|True), 'versionsuffix': '', 'toolchain': ''}
         Iterable builddependencies are flattened when not iterating.
 
         :param build_only: only return build dependencies, discard others
+        :param runtime_only: only return runtime dependencies, discard others
         """
-        deps = self.builddependencies()
+        if runtime_only:
+            deps = []
+        else:
+            deps = self.builddependencies()
 
         if not build_only:
             # use += rather than .extend to get a new list rather than updating list of build deps in place...

From 46a52dffb6cac36aa0403401dfec1119d6792018 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Mikael=20=C3=96hman?= <micketeer@gmail.com>
Date: Thu, 15 Aug 2024 20:28:02 +0200
Subject: [PATCH 11/25] Switch to more reliable way of checking for direct
 dependencies.

---
 easybuild/framework/easyblock.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py
index 0397d74b94..42ae536487 100644
--- a/easybuild/framework/easyblock.py
+++ b/easybuild/framework/easyblock.py
@@ -1451,8 +1451,8 @@ def make_module_extra(self, altroot=None, altversion=None):
                                  for path in glob.glob(f'{self.installdir}/lib64/python*/site-packages')
                                  if re.match(self.installdir + r'/lib64/python\d+\.\d+/site-packages', path)]
 
-            use_ebpythonprefixes = get_software_root('Python') and build_option('prefer_ebpythonprefixes') and \
-                self.cfg['prefer_ebpythonprefixes']
+            use_ebpythonprefixes = 'Python' in self.cfg.dependencies(runtime_only=True) and \
+                build_option('prefer_ebpythonprefixes') and self.cfg['prefer_ebpythonprefixes']
 
             if len(python_paths) > 1 and not use_ebpythonprefixes:
                 raise EasyBuildError('Multiple python paths requires EBPYTHONPREFIXES: ' + ', '.join(python_paths))

From fb51411436212ed81987427d1d62c5228f1e87ed Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Mikael=20=C3=96hman?= <micketeer@gmail.com>
Date: Thu, 15 Aug 2024 20:59:07 +0200
Subject: [PATCH 12/25] Drop extra check for multiple PYTHONPATH's

---
 easybuild/framework/easyblock.py | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py
index 42ae536487..5b77cc5d0d 100644
--- a/easybuild/framework/easyblock.py
+++ b/easybuild/framework/easyblock.py
@@ -1454,9 +1454,7 @@ def make_module_extra(self, altroot=None, altversion=None):
             use_ebpythonprefixes = 'Python' in self.cfg.dependencies(runtime_only=True) and \
                 build_option('prefer_ebpythonprefixes') and self.cfg['prefer_ebpythonprefixes']
 
-            if len(python_paths) > 1 and not use_ebpythonprefixes:
-                raise EasyBuildError('Multiple python paths requires EBPYTHONPREFIXES: ' + ', '.join(python_paths))
-            elif python_paths:
+            if python_paths:
                 # Add paths unless they were already added
                 if use_ebpythonprefixes:
                     if '' not in self.module_generator.added_paths_per_key['EBPYTHONPREFIXES']:

From a0640063708bdf981040995539bc44bf0e2f528b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Mikael=20=C3=96hman?= <micketeer@gmail.com>
Date: Thu, 15 Aug 2024 21:09:17 +0200
Subject: [PATCH 13/25] Fix check for Python runtime dep

---
 easybuild/framework/easyblock.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py
index 5b77cc5d0d..44b1c6d155 100644
--- a/easybuild/framework/easyblock.py
+++ b/easybuild/framework/easyblock.py
@@ -1451,7 +1451,8 @@ def make_module_extra(self, altroot=None, altversion=None):
                                  for path in glob.glob(f'{self.installdir}/lib64/python*/site-packages')
                                  if re.match(self.installdir + r'/lib64/python\d+\.\d+/site-packages', path)]
 
-            use_ebpythonprefixes = 'Python' in self.cfg.dependencies(runtime_only=True) and \
+            runtime_deps = [dep['name'] for dep in self.cfg.dependencies(runtime_only=True)]
+            use_ebpythonprefixes = 'Python' in runtime_deps and \
                 build_option('prefer_ebpythonprefixes') and self.cfg['prefer_ebpythonprefixes']
 
             if python_paths:

From d281fc8c542006730711c0908a8c477a86161514 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Mikael=20=C3=96hman?= <micketeer@gmail.com>
Date: Fri, 16 Aug 2024 12:17:49 +0200
Subject: [PATCH 14/25] Switch method of detecting base python install

---
 easybuild/framework/easyblock.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py
index 44b1c6d155..9f7ce4a282 100644
--- a/easybuild/framework/easyblock.py
+++ b/easybuild/framework/easyblock.py
@@ -1439,7 +1439,7 @@ def make_module_extra(self, altroot=None, altversion=None):
             lines.append(self.module_generator.append_paths(key, value, allow_abs=self.cfg['allow_append_abs_path']))
 
         # Add automatic PYTHONPATH or EBPYTHONPREFIXES if they aren't already present and python paths exist
-        if self.name not in ['Python', 'Miniconda', 'Anaconda']:  # nothing extra needed for base python installations
+        if not os.path.isfile(f'{self.installdir}/bin/python'):  # only needed when not a base python installation
             # Exclude symlinks lib -> lib64 or vice verse to avoid duplicates
             python_paths = []
             if not os.path.islink(f'{self.installdir}/lib'):

From 11cffaa8b8ee6e7a2ae0890f3149383623c4b015 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Mikael=20=C3=96hman?= <micketeer@gmail.com>
Date: Fri, 16 Aug 2024 12:24:47 +0200
Subject: [PATCH 15/25] Add all the found python paths even when using
 PYTHONPATH

---
 easybuild/framework/easyblock.py | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py
index 9f7ce4a282..71bc79d3bf 100644
--- a/easybuild/framework/easyblock.py
+++ b/easybuild/framework/easyblock.py
@@ -1460,8 +1460,10 @@ def make_module_extra(self, altroot=None, altversion=None):
                 if use_ebpythonprefixes:
                     if '' not in self.module_generator.added_paths_per_key['EBPYTHONPREFIXES']:
                         lines.append(self.module_generator.prepend_paths('EBPYTHONPREFIXES', ''))
-                elif python_paths[0] not in self.module_generator.added_paths_per_key['PYTHONPATH']:
-                    lines.append(self.module_generator.prepend_paths('PYTHONPATH', python_paths))
+                else:
+                    for python_path in python_paths:
+                        if python_path not in self.module_generator.added_paths_per_key['PYTHONPATH']:
+                            lines.append(self.module_generator.prepend_paths('PYTHONPATH', python_path))
 
         modloadmsg = self.cfg['modloadmsg']
         if modloadmsg:

From f90a12bcf5661858e7a84078df3e6e60d3285ac3 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Mikael=20=C3=96hman?= <micketeer@gmail.com>
Date: Fri, 16 Aug 2024 13:14:13 +0200
Subject: [PATCH 16/25] Drop lib64 variant as it's not supported by
 sitecustomize.py

Our sitecustomize.py only considers the lib/pythonx.xx/site-packages
paths anyway, so we must not consider any more when adding the automatic
paths.
---
 easybuild/framework/easyblock.py | 13 +++----------
 1 file changed, 3 insertions(+), 10 deletions(-)

diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py
index 71bc79d3bf..baf7e93446 100644
--- a/easybuild/framework/easyblock.py
+++ b/easybuild/framework/easyblock.py
@@ -1440,16 +1440,9 @@ def make_module_extra(self, altroot=None, altversion=None):
 
         # Add automatic PYTHONPATH or EBPYTHONPREFIXES if they aren't already present and python paths exist
         if not os.path.isfile(f'{self.installdir}/bin/python'):  # only needed when not a base python installation
-            # Exclude symlinks lib -> lib64 or vice verse to avoid duplicates
-            python_paths = []
-            if not os.path.islink(f'{self.installdir}/lib'):
-                python_paths += [path[len(self.installdir)+1:]
-                                 for path in glob.glob(f'{self.installdir}/lib/python*/site-packages')
-                                 if re.match(self.installdir + r'/lib/python\d+\.\d+/site-packages', path)]
-            if not os.path.islink(f'{self.installdir}/lib64'):
-                python_paths += [path[len(self.installdir)+1:]
-                                 for path in glob.glob(f'{self.installdir}/lib64/python*/site-packages')
-                                 if re.match(self.installdir + r'/lib64/python\d+\.\d+/site-packages', path)]
+            python_paths = [path[len(self.installdir)+1:]
+                            for path in glob.glob(f'{self.installdir}/lib/python*/site-packages')
+                            if re.match(self.installdir + r'/lib/python\d+\.\d+/site-packages', path)]
 
             runtime_deps = [dep['name'] for dep in self.cfg.dependencies(runtime_only=True)]
             use_ebpythonprefixes = 'Python' in runtime_deps and \

From 1d78e6c2c2ac4777e060a24827718457e23075af Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Mikael=20=C3=96hman?= <micketeer@gmail.com>
Date: Fri, 16 Aug 2024 15:20:47 +0200
Subject: [PATCH 17/25] Default --prefer-ebpythonprefixes to false

---
 easybuild/tools/options.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/easybuild/tools/options.py b/easybuild/tools/options.py
index c5fc72061d..551cb14a58 100644
--- a/easybuild/tools/options.py
+++ b/easybuild/tools/options.py
@@ -491,7 +491,7 @@ def override_options(self):
             'pre-create-installdir': ("Create installation directory before submitting build jobs",
                                       None, 'store_true', True),
             'prefer-ebpythonprefixes': (("Prefer EBPYTHONPREFIXES over PYTHONPATH when possible"),
-                                        None, 'store_true', True),
+                                        None, 'store_true', False),
             'pretend': (("Does the build/installation in a test directory located in $HOME/easybuildinstall"),
                         None, 'store_true', False, 'p'),
             'read-only-installdir': ("Set read-only permissions on installation directory after installation",

From 7b7a59f756d94edf1ae49a2e3d32b23e06cbd71e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Mikael=20=C3=96hman?= <micketeer@gmail.com>
Date: Thu, 19 Sep 2024 18:29:40 +0200
Subject: [PATCH 18/25] Rename options and parameters for ebpythonprefixes
 change

---
 easybuild/framework/easyblock.py          | 13 +++++++------
 easybuild/framework/easyconfig/default.py |  2 +-
 easybuild/tools/config.py                 |  8 +++++++-
 easybuild/tools/options.py                |  7 +++++--
 4 files changed, 20 insertions(+), 10 deletions(-)

diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py
index baf7e93446..1538334eb6 100644
--- a/easybuild/framework/easyblock.py
+++ b/easybuild/framework/easyblock.py
@@ -72,7 +72,7 @@
 from easybuild.tools.build_details import get_build_stats
 from easybuild.tools.build_log import EasyBuildError, EasyBuildExit, dry_run_msg, dry_run_warning, dry_run_set_dirs
 from easybuild.tools.build_log import print_error, print_msg, print_warning
-from easybuild.tools.config import CHECKSUM_PRIORITY_JSON, DEFAULT_ENVVAR_USERS_MODULES
+from easybuild.tools.config import CHECKSUM_PRIORITY_JSON, DEFAULT_ENVVAR_USERS_MODULES, PYTHONPATH, EBPYTHONPREFIXES
 from easybuild.tools.config import FORCE_DOWNLOAD_ALL, FORCE_DOWNLOAD_PATCHES, FORCE_DOWNLOAD_SOURCES
 from easybuild.tools.config import EASYBUILD_SOURCES_URL # noqa
 from easybuild.tools.config import build_option, build_path, get_log_filename, get_repository, get_repositorypath
@@ -1446,17 +1446,18 @@ def make_module_extra(self, altroot=None, altversion=None):
 
             runtime_deps = [dep['name'] for dep in self.cfg.dependencies(runtime_only=True)]
             use_ebpythonprefixes = 'Python' in runtime_deps and \
-                build_option('prefer_ebpythonprefixes') and self.cfg['prefer_ebpythonprefixes']
+                build_option('prefer_python_search_path') == EBPYTHONPREFIXES and not self.cfg['force_pythonpath']
 
             if python_paths:
                 # Add paths unless they were already added
                 if use_ebpythonprefixes:
-                    if '' not in self.module_generator.added_paths_per_key['EBPYTHONPREFIXES']:
-                        lines.append(self.module_generator.prepend_paths('EBPYTHONPREFIXES', ''))
+                    path = ''  # EBPYTHONPREFIXES are relative to the install dir
+                    if path not in self.module_generator.added_paths_per_key[EBPYTHONPREFIXES]:
+                        lines.append(self.module_generator.prepend_paths(EBPYTHONPREFIXES, path))
                 else:
                     for python_path in python_paths:
-                        if python_path not in self.module_generator.added_paths_per_key['PYTHONPATH']:
-                            lines.append(self.module_generator.prepend_paths('PYTHONPATH', python_path))
+                        if python_path not in self.module_generator.added_paths_per_key[PYTHONPATH]:
+                            lines.append(self.module_generator.prepend_paths(PYTHONPATH, python_path))
 
         modloadmsg = self.cfg['modloadmsg']
         if modloadmsg:
diff --git a/easybuild/framework/easyconfig/default.py b/easybuild/framework/easyconfig/default.py
index 43f6093588..db1e12c324 100644
--- a/easybuild/framework/easyconfig/default.py
+++ b/easybuild/framework/easyconfig/default.py
@@ -115,7 +115,7 @@
     'patches': [[], "List of patches to apply", BUILD],
     'prebuildopts': ['', 'Extra options pre-passed to build command.', BUILD],
     'preconfigopts': ['', 'Extra options pre-passed to configure.', BUILD],
-    'prefer_ebpythonprefixes': [True, "Prefer EBPYTHONPREFIXES over PYTHONPATH when possible.", BUILD],
+    'force_pythonpath': [False, "Force use of PYTHONPATH for python seearch path when possible.", BUILD],
     'preinstallopts': ['', 'Extra prefix options for installation.', BUILD],
     'pretestopts': ['', 'Extra prefix options for test.', BUILD],
     'postinstallcmds': [[], 'Commands to run after the install step.', BUILD],
diff --git a/easybuild/tools/config.py b/easybuild/tools/config.py
index 67473b4dc0..384ee065d3 100644
--- a/easybuild/tools/config.py
+++ b/easybuild/tools/config.py
@@ -175,6 +175,10 @@
 OUTPUT_STYLE_RICH = 'rich'
 OUTPUT_STYLES = (OUTPUT_STYLE_AUTO, OUTPUT_STYLE_BASIC, OUTPUT_STYLE_NO_COLOR, OUTPUT_STYLE_RICH)
 
+PYTHONPATH = 'PYTHONPATH'
+EBPYTHONPREFIXES = 'EBPYTHONPREFIXES'
+PYTHON_SEARCH_PATH_TYPES = [PYTHONPATH, EBPYTHONPREFIXES]
+
 
 class Singleton(ABCMeta):
     """Serves as metaclass for classes that should implement the Singleton pattern.
@@ -336,7 +340,6 @@ def mk_full_default_path(name, prefix=DEFAULT_PREFIX):
         'modules_tool_version_check',
         'mpi_tests',
         'pre_create_installdir',
-        'prefer_ebpythonprefixes',
         'show_progress_bar',
         'trace',
     ],
@@ -408,6 +411,9 @@ def mk_full_default_path(name, prefix=DEFAULT_PREFIX):
     OUTPUT_STYLE_AUTO: [
         'output_style',
     ],
+    PYTHONPATH: [
+        'prefer_python_search_path',
+    ]
 }
 # build option that do not have a perfectly matching command line option
 BUILD_OPTIONS_OTHER = {
diff --git a/easybuild/tools/options.py b/easybuild/tools/options.py
index 551cb14a58..6a39bfe80e 100644
--- a/easybuild/tools/options.py
+++ b/easybuild/tools/options.py
@@ -78,6 +78,7 @@
 from easybuild.tools.config import OUTPUT_STYLE_AUTO, OUTPUT_STYLES, WARN
 from easybuild.tools.config import get_pretend_installpath, init, init_build_options, mk_full_default_path
 from easybuild.tools.config import BuildOptions, ConfigurationVariables
+from easybuild.tools.config import PYTHON_SEARCH_PATH_TYPES, PYTHONPATH
 from easybuild.tools.configobj import ConfigObj, ConfigObjError
 from easybuild.tools.docs import FORMAT_JSON, FORMAT_MD, FORMAT_RST, FORMAT_TXT
 from easybuild.tools.docs import avail_cfgfile_constants, avail_easyconfig_constants, avail_easyconfig_licenses
@@ -490,8 +491,10 @@ def override_options(self):
                                             None, 'store_true', False),
             'pre-create-installdir': ("Create installation directory before submitting build jobs",
                                       None, 'store_true', True),
-            'prefer-ebpythonprefixes': (("Prefer EBPYTHONPREFIXES over PYTHONPATH when possible"),
-                                        None, 'store_true', False),
+            'prefer-python-search-path': (("Prefer using specified environment variable when possible to specify where"
+                                           " Python packages were installed; see also "
+                                           "https://docs.easybuild.io/python-search-path"),
+                                          'choice', 'store_or_None', PYTHONPATH, PYTHON_SEARCH_PATH_TYPES),
             'pretend': (("Does the build/installation in a test directory located in $HOME/easybuildinstall"),
                         None, 'store_true', False, 'p'),
             'read-only-installdir': ("Set read-only permissions on installation directory after installation",

From dc488f5da8a3b0038940a9e740f563193efc286b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Mikael=20=C3=96hman?= <micketeer@gmail.com>
Date: Mon, 23 Sep 2024 16:41:05 +0200
Subject: [PATCH 19/25] Switch to using relpath to make code robust

---
 easybuild/framework/easyblock.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py
index 1538334eb6..f63c15fffd 100644
--- a/easybuild/framework/easyblock.py
+++ b/easybuild/framework/easyblock.py
@@ -1440,7 +1440,7 @@ def make_module_extra(self, altroot=None, altversion=None):
 
         # Add automatic PYTHONPATH or EBPYTHONPREFIXES if they aren't already present and python paths exist
         if not os.path.isfile(f'{self.installdir}/bin/python'):  # only needed when not a base python installation
-            python_paths = [path[len(self.installdir)+1:]
+            python_paths = [os.path.relpath(path, self.installdir)
                             for path in glob.glob(f'{self.installdir}/lib/python*/site-packages')
                             if re.match(self.installdir + r'/lib/python\d+\.\d+/site-packages', path)]
 

From 7e417d3f3d532c6ed19bf006dd14569a3d5571ca Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Mikael=20=C3=96hman?= <micketeer@gmail.com>
Date: Fri, 27 Sep 2024 01:05:02 +0200
Subject: [PATCH 20/25] Use relative paths when looking for python paths

Not only simplifies the code but is also the obviously correct way
which i should have used from the start.
---
 easybuild/framework/easyblock.py | 7 +++----
 1 file changed, 3 insertions(+), 4 deletions(-)

diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py
index f63c15fffd..a6da6ce63a 100644
--- a/easybuild/framework/easyblock.py
+++ b/easybuild/framework/easyblock.py
@@ -1439,10 +1439,9 @@ def make_module_extra(self, altroot=None, altversion=None):
             lines.append(self.module_generator.append_paths(key, value, allow_abs=self.cfg['allow_append_abs_path']))
 
         # Add automatic PYTHONPATH or EBPYTHONPREFIXES if they aren't already present and python paths exist
-        if not os.path.isfile(f'{self.installdir}/bin/python'):  # only needed when not a base python installation
-            python_paths = [os.path.relpath(path, self.installdir)
-                            for path in glob.glob(f'{self.installdir}/lib/python*/site-packages')
-                            if re.match(self.installdir + r'/lib/python\d+\.\d+/site-packages', path)]
+        if not os.path.isfile(os.path.join(self.installdir, 'bin/python')):  # only needed when not a python install
+            python_paths = [path for path in glob.glob('lib/python*/site-packages', root_dir=self.installdir)
+                            if re.match(r'lib/python\d+\.\d+/site-packages', path)]
 
             runtime_deps = [dep['name'] for dep in self.cfg.dependencies(runtime_only=True)]
             use_ebpythonprefixes = 'Python' in runtime_deps and \

From 9813bc8e3cef36aed320ab6ff36a567f9385a976 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Mikael=20=C3=96hman?= <micketeer@gmail.com>
Date: Fri, 27 Sep 2024 12:04:09 +0200
Subject: [PATCH 21/25] Replace glob(root_dir=...) which only is supported in
 python 3.10

Instead, using absolute paths and then make them relative.
---
 easybuild/framework/easyblock.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py
index a6da6ce63a..65ae805380 100644
--- a/easybuild/framework/easyblock.py
+++ b/easybuild/framework/easyblock.py
@@ -1440,8 +1440,9 @@ def make_module_extra(self, altroot=None, altversion=None):
 
         # Add automatic PYTHONPATH or EBPYTHONPREFIXES if they aren't already present and python paths exist
         if not os.path.isfile(os.path.join(self.installdir, 'bin/python')):  # only needed when not a python install
-            python_paths = [path for path in glob.glob('lib/python*/site-packages', root_dir=self.installdir)
-                            if re.match(r'lib/python\d+\.\d+/site-packages', path)]
+            candidate_paths = (os.path.relpath(path, self.installdir)
+                               for path in glob.glob(f'{self.installdir}/lib/python*/site-packages'))
+            python_paths = [path for path in candidate_paths if re.match(r'lib/python\d+\.\d+/site-packages', path)]
 
             runtime_deps = [dep['name'] for dep in self.cfg.dependencies(runtime_only=True)]
             use_ebpythonprefixes = 'Python' in runtime_deps and \

From 206736eb69ddf1e497b3ac07fb846d07bde13cca Mon Sep 17 00:00:00 2001
From: Kenneth Hoste <kenneth.hoste@ugent.be>
Date: Mon, 30 Sep 2024 11:03:54 +0200
Subject: [PATCH 22/25] add test to verify that $PYTHONPATH or
 $EBPYTHONPREFIXES are set correctly by generated module file

---
 test/framework/toy_build.py | 63 +++++++++++++++++++++++++++++++++++++
 1 file changed, 63 insertions(+)

diff --git a/test/framework/toy_build.py b/test/framework/toy_build.py
index afe0b86bef..76ef640b29 100644
--- a/test/framework/toy_build.py
+++ b/test/framework/toy_build.py
@@ -4238,6 +4238,69 @@ def test_eb_error(self):
             stderr = stderr.getvalue()
             self.assertTrue(regex.search(stderr), f"Pattern '{regex.pattern}' should be found in {stderr}")
 
+    def test_toy_python(self):
+        """
+        Test whether $PYTHONPATH or $EBPYTHONPREFIXES are set correctly.
+        """
+        # generate fake Python module that we can use as runtime dependency for toy
+        # (required condition for use of $EBPYTHONPREFIXES)
+        fake_mods_path = os.path.join(self.test_prefix, 'modules')
+        fake_python_mod = os.path.join(fake_mods_path, 'Python', '3.6')
+        if get_module_syntax() == 'Lua':
+            fake_python_mod += '.lua'
+            write_file(fake_python_mod, '')
+        else:
+            write_file(fake_python_mod, '#%Module')
+        self.modtool.use(fake_mods_path)
+
+        test_ecs = os.path.join(os.path.dirname(__file__), 'easyconfigs', 'test_ecs')
+        toy_ec = os.path.join(test_ecs, 't', 'toy', 'toy-0.0.eb')
+
+        test_ec_txt = read_file(toy_ec)
+        test_ec_txt += "\npostinstallcmds.append('mkdir -p %(installdir)s/lib/python3.6/site-packages')"
+        test_ec_txt += "\npostinstallcmds.append('touch %(installdir)s/lib/python3.6/site-packages/foo.py')"
+
+        test_ec = os.path.join(self.test_prefix, 'test.eb')
+        write_file(test_ec, test_ec_txt)
+        self.run_test_toy_build_with_output(ec_file=test_ec)
+
+        toy_mod = os.path.join(self.test_installpath, 'modules', 'all', 'toy', '0.0')
+        if get_module_syntax() == 'Lua':
+            toy_mod += '.lua'
+        toy_mod_txt = read_file(toy_mod)
+
+        pythonpath_regex = re.compile('^prepend.path.*PYTHONPATH.*lib/python3.6/site-packages', re.M)
+
+        self.assertTrue(pythonpath_regex.search(toy_mod_txt),
+                        f"Pattern '{pythonpath_regex.pattern}' found in: {toy_mod_txt}")
+
+        # also check when opting in to use $EBPYTHONPREFIXES instead of $PYTHONPATH
+        args = ['--prefer-python-search-path=EBPYTHONPREFIXES']
+        self.run_test_toy_build_with_output(ec_file=test_ec, extra_args=args)
+        toy_mod_txt = read_file(toy_mod)
+        # if Python is not listed as a runtime dependency then $PYTHONPATH is still used,
+        # because the Python dependency used must be aware of $EBPYTHONPREFIXES
+        # (see sitecustomize.py installed by Python easyblock)
+        self.assertTrue(pythonpath_regex.search(toy_mod_txt),
+                        f"Pattern '{pythonpath_regex.pattern}' found in: {toy_mod_txt}")
+
+        test_ec_txt += "\ndependencies = [('Python', '3.6', '', SYSTEM)]"
+        write_file(test_ec, test_ec_txt)
+        self.run_test_toy_build_with_output(ec_file=test_ec, extra_args=args)
+        toy_mod_txt = read_file(toy_mod)
+
+        ebpythonprefixes_regex = re.compile('^prepend.path.*EBPYTHONPREFIXES.*root', re.M)
+        self.assertTrue(ebpythonprefixes_regex.search(toy_mod_txt),
+                        f"Pattern '{ebpythonprefixes_regex.pattern}' found in: {toy_mod_txt}")
+
+        test_ec_txt += "\nforce_pythonpath = True"
+        write_file(test_ec, test_ec_txt)
+        self.run_test_toy_build_with_output(ec_file=test_ec)
+        toy_mod_txt = read_file(toy_mod)
+
+        self.assertTrue(pythonpath_regex.search(toy_mod_txt),
+                        f"Pattern '{pythonpath_regex.pattern}' found in: {toy_mod_txt}")
+
 
 def suite():
     """ return all the tests in this file """

From 5eb6d078d9a21bbeab7bf3cf31ad213056d76e22 Mon Sep 17 00:00:00 2001
From: Kenneth Hoste <kenneth.hoste@ugent.be>
Date: Mon, 30 Sep 2024 11:09:48 +0200
Subject: [PATCH 23/25] move logic to set $PYTHONPATH or $EBPYTHONPREFIX to
 dedicated make_module_pythonpath method

---
 easybuild/framework/easyblock.py | 53 ++++++++++++++++++++------------
 1 file changed, 33 insertions(+), 20 deletions(-)

diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py
index 65ae805380..87e5660e2b 100644
--- a/easybuild/framework/easyblock.py
+++ b/easybuild/framework/easyblock.py
@@ -1390,6 +1390,37 @@ def make_module_description(self):
         """
         return self.module_generator.get_description()
 
+    def make_module_pythonpath(self):
+        """
+        Add lines for module file to update $PYTHONPATH or $EBPYTHONPREFIXES,
+        if they aren't already present and the standard lib/python*/site-packages subdirectory exists
+        """
+        lines = []
+        if not os.path.isfile(os.path.join(self.installdir, 'bin', 'python')):  # only needed when not a python install
+            python_subdir_pattern = os.path.join(self.installdir, 'lib', 'python*', 'site-packages')
+            candidate_paths = (os.path.relpath(path, self.installdir) for path in glob.glob(python_subdir_pattern))
+            python_paths = [path for path in candidate_paths if re.match(r'lib/python\d+\.\d+/site-packages', path)]
+
+            runtime_deps = [dep['name'] for dep in self.cfg.dependencies(runtime_only=True)]
+            use_ebpythonprefixes = all([
+                'Python' in runtime_deps,
+                build_option('prefer_python_search_path') == EBPYTHONPREFIXES,
+                not self.cfg['force_pythonpath']
+            ])
+
+            if python_paths:
+                # add paths unless they were already added
+                if use_ebpythonprefixes:
+                    path = ''  # EBPYTHONPREFIXES are relative to the install dir
+                    if path not in self.module_generator.added_paths_per_key[EBPYTHONPREFIXES]:
+                        lines.append(self.module_generator.prepend_paths(EBPYTHONPREFIXES, path))
+                else:
+                    for python_path in python_paths:
+                        if python_path not in self.module_generator.added_paths_per_key[PYTHONPATH]:
+                            lines.append(self.module_generator.prepend_paths(PYTHONPATH, python_path))
+
+        return lines
+
     def make_module_extra(self, altroot=None, altversion=None):
         """
         Set extra stuff in module file, e.g. $EBROOT*, $EBVERSION*, etc.
@@ -1438,26 +1469,8 @@ def make_module_extra(self, altroot=None, altversion=None):
                                      value, type(value))
             lines.append(self.module_generator.append_paths(key, value, allow_abs=self.cfg['allow_append_abs_path']))
 
-        # Add automatic PYTHONPATH or EBPYTHONPREFIXES if they aren't already present and python paths exist
-        if not os.path.isfile(os.path.join(self.installdir, 'bin/python')):  # only needed when not a python install
-            candidate_paths = (os.path.relpath(path, self.installdir)
-                               for path in glob.glob(f'{self.installdir}/lib/python*/site-packages'))
-            python_paths = [path for path in candidate_paths if re.match(r'lib/python\d+\.\d+/site-packages', path)]
-
-            runtime_deps = [dep['name'] for dep in self.cfg.dependencies(runtime_only=True)]
-            use_ebpythonprefixes = 'Python' in runtime_deps and \
-                build_option('prefer_python_search_path') == EBPYTHONPREFIXES and not self.cfg['force_pythonpath']
-
-            if python_paths:
-                # Add paths unless they were already added
-                if use_ebpythonprefixes:
-                    path = ''  # EBPYTHONPREFIXES are relative to the install dir
-                    if path not in self.module_generator.added_paths_per_key[EBPYTHONPREFIXES]:
-                        lines.append(self.module_generator.prepend_paths(EBPYTHONPREFIXES, path))
-                else:
-                    for python_path in python_paths:
-                        if python_path not in self.module_generator.added_paths_per_key[PYTHONPATH]:
-                            lines.append(self.module_generator.prepend_paths(PYTHONPATH, python_path))
+        # add lines to update $PYTHONPATH or $EBPYTHONPREFIXES
+        lines.extend(self.make_module_pythonpath())
 
         modloadmsg = self.cfg['modloadmsg']
         if modloadmsg:

From 76fec7b84747f036c83b85238499c4a303b73285 Mon Sep 17 00:00:00 2001
From: Kenneth Hoste <kenneth.hoste@ugent.be>
Date: Mon, 30 Sep 2024 15:28:03 +0200
Subject: [PATCH 24/25] take into account multi_deps when determining whether
 to use $PYTHONPATH or $EBPYTHONPREFIXES

---
 easybuild/framework/easyblock.py | 25 ++++++++++++++++++++-----
 test/framework/toy_build.py      | 30 +++++++++++++++++++-----------
 2 files changed, 39 insertions(+), 16 deletions(-)

diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py
index 87e5660e2b..0dc1e2e2a5 100644
--- a/easybuild/framework/easyblock.py
+++ b/easybuild/framework/easyblock.py
@@ -1401,12 +1401,27 @@ def make_module_pythonpath(self):
             candidate_paths = (os.path.relpath(path, self.installdir) for path in glob.glob(python_subdir_pattern))
             python_paths = [path for path in candidate_paths if re.match(r'lib/python\d+\.\d+/site-packages', path)]
 
+            # determine whether Python is a runtime dependency;
+            # if so, we assume it was installed with EasyBuild, and hence is aware of $EBPYTHONPREFIXES
             runtime_deps = [dep['name'] for dep in self.cfg.dependencies(runtime_only=True)]
-            use_ebpythonprefixes = all([
-                'Python' in runtime_deps,
-                build_option('prefer_python_search_path') == EBPYTHONPREFIXES,
-                not self.cfg['force_pythonpath']
-            ])
+
+            # don't use $EBPYTHONPREFIXES unless we can and it's preferred or necesary (due to use of multi_deps)
+            use_ebpythonprefixes = False
+            multi_deps = self.cfg['multi_deps']
+
+            if self.cfg['force_pythonpath']:
+                self.log.info("Using $PYTHONPATH to specify Python search path, since 'force_pythonpath' is set")
+
+            elif 'Python' in runtime_deps:
+                self.log.info("Found Python runtime dependency, so considering $EBPYTHONPREFIXES...")
+
+                if build_option('prefer_python_search_path') == EBPYTHONPREFIXES:
+                    self.log.info("Preferred Python search path is $EBPYTHONPREFIXES, so using that")
+                    use_ebpythonprefixes = True
+
+            elif multi_deps and 'Python' in multi_deps:
+                self.log.info("Python is listed in 'multi_deps', so using $EBPYTHONPREFIXES instead of $PYTHONPATH")
+                use_ebpythonprefixes = True
 
             if python_paths:
                 # add paths unless they were already added
diff --git a/test/framework/toy_build.py b/test/framework/toy_build.py
index 76ef640b29..e9b857ecee 100644
--- a/test/framework/toy_build.py
+++ b/test/framework/toy_build.py
@@ -4242,15 +4242,16 @@ def test_toy_python(self):
         """
         Test whether $PYTHONPATH or $EBPYTHONPREFIXES are set correctly.
         """
-        # generate fake Python module that we can use as runtime dependency for toy
+        # generate fake Python modules that we can use as runtime dependency for toy
         # (required condition for use of $EBPYTHONPREFIXES)
         fake_mods_path = os.path.join(self.test_prefix, 'modules')
-        fake_python_mod = os.path.join(fake_mods_path, 'Python', '3.6')
-        if get_module_syntax() == 'Lua':
-            fake_python_mod += '.lua'
-            write_file(fake_python_mod, '')
-        else:
-            write_file(fake_python_mod, '#%Module')
+        for pyver in ('2.7', '3.6'):
+            fake_python_mod = os.path.join(fake_mods_path, 'Python', pyver)
+            if get_module_syntax() == 'Lua':
+                fake_python_mod += '.lua'
+                write_file(fake_python_mod, '')
+            else:
+                write_file(fake_python_mod, '#%Module')
         self.modtool.use(fake_mods_path)
 
         test_ecs = os.path.join(os.path.dirname(__file__), 'easyconfigs', 'test_ecs')
@@ -4284,8 +4285,8 @@ def test_toy_python(self):
         self.assertTrue(pythonpath_regex.search(toy_mod_txt),
                         f"Pattern '{pythonpath_regex.pattern}' found in: {toy_mod_txt}")
 
-        test_ec_txt += "\ndependencies = [('Python', '3.6', '', SYSTEM)]"
-        write_file(test_ec, test_ec_txt)
+        # if Python is listed as runtime dependency, then $EBPYTHONPREFIXES is used if it's preferred
+        write_file(test_ec, test_ec_txt + "\ndependencies = [('Python', '3.6', '', SYSTEM)]")
         self.run_test_toy_build_with_output(ec_file=test_ec, extra_args=args)
         toy_mod_txt = read_file(toy_mod)
 
@@ -4293,11 +4294,18 @@ def test_toy_python(self):
         self.assertTrue(ebpythonprefixes_regex.search(toy_mod_txt),
                         f"Pattern '{ebpythonprefixes_regex.pattern}' found in: {toy_mod_txt}")
 
-        test_ec_txt += "\nforce_pythonpath = True"
-        write_file(test_ec, test_ec_txt)
+        # if Python is listed in multi_deps, then $EBPYTHONPREFIXES is used, even if it's not explicitely preferred
+        write_file(test_ec, test_ec_txt + "\nmulti_deps = {'Python': ['2.7', '3.6']}")
         self.run_test_toy_build_with_output(ec_file=test_ec)
         toy_mod_txt = read_file(toy_mod)
 
+        self.assertTrue(ebpythonprefixes_regex.search(toy_mod_txt),
+                        f"Pattern '{ebpythonprefixes_regex.pattern}' found in: {toy_mod_txt}")
+
+        write_file(test_ec, test_ec_txt + "\ndependencies = [('Python', '3.6', '', SYSTEM)]\nforce_pythonpath = True")
+        self.run_test_toy_build_with_output(ec_file=test_ec, extra_args=args)
+        toy_mod_txt = read_file(toy_mod)
+
         self.assertTrue(pythonpath_regex.search(toy_mod_txt),
                         f"Pattern '{pythonpath_regex.pattern}' found in: {toy_mod_txt}")
 

From 11f132ea0ee733a849f715daaed89498c8763f06 Mon Sep 17 00:00:00 2001
From: Kenneth Hoste <kenneth.hoste@ugent.be>
Date: Mon, 30 Sep 2024 16:03:48 +0200
Subject: [PATCH 25/25] remove force_pythonpath easyconfig paramter, can be
 re-introduced later if there's a real need for it

---
 easybuild/framework/easyblock.py          | 5 +----
 easybuild/framework/easyconfig/default.py | 1 -
 test/framework/toy_build.py               | 7 -------
 3 files changed, 1 insertion(+), 12 deletions(-)

diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py
index 0dc1e2e2a5..a786cbea2b 100644
--- a/easybuild/framework/easyblock.py
+++ b/easybuild/framework/easyblock.py
@@ -1409,10 +1409,7 @@ def make_module_pythonpath(self):
             use_ebpythonprefixes = False
             multi_deps = self.cfg['multi_deps']
 
-            if self.cfg['force_pythonpath']:
-                self.log.info("Using $PYTHONPATH to specify Python search path, since 'force_pythonpath' is set")
-
-            elif 'Python' in runtime_deps:
+            if 'Python' in runtime_deps:
                 self.log.info("Found Python runtime dependency, so considering $EBPYTHONPREFIXES...")
 
                 if build_option('prefer_python_search_path') == EBPYTHONPREFIXES:
diff --git a/easybuild/framework/easyconfig/default.py b/easybuild/framework/easyconfig/default.py
index db1e12c324..438bcea4e1 100644
--- a/easybuild/framework/easyconfig/default.py
+++ b/easybuild/framework/easyconfig/default.py
@@ -115,7 +115,6 @@
     'patches': [[], "List of patches to apply", BUILD],
     'prebuildopts': ['', 'Extra options pre-passed to build command.', BUILD],
     'preconfigopts': ['', 'Extra options pre-passed to configure.', BUILD],
-    'force_pythonpath': [False, "Force use of PYTHONPATH for python seearch path when possible.", BUILD],
     'preinstallopts': ['', 'Extra prefix options for installation.', BUILD],
     'pretestopts': ['', 'Extra prefix options for test.', BUILD],
     'postinstallcmds': [[], 'Commands to run after the install step.', BUILD],
diff --git a/test/framework/toy_build.py b/test/framework/toy_build.py
index e9b857ecee..edd9e94fcc 100644
--- a/test/framework/toy_build.py
+++ b/test/framework/toy_build.py
@@ -4302,13 +4302,6 @@ def test_toy_python(self):
         self.assertTrue(ebpythonprefixes_regex.search(toy_mod_txt),
                         f"Pattern '{ebpythonprefixes_regex.pattern}' found in: {toy_mod_txt}")
 
-        write_file(test_ec, test_ec_txt + "\ndependencies = [('Python', '3.6', '', SYSTEM)]\nforce_pythonpath = True")
-        self.run_test_toy_build_with_output(ec_file=test_ec, extra_args=args)
-        toy_mod_txt = read_file(toy_mod)
-
-        self.assertTrue(pythonpath_regex.search(toy_mod_txt),
-                        f"Pattern '{pythonpath_regex.pattern}' found in: {toy_mod_txt}")
-
 
 def suite():
     """ return all the tests in this file """