update easyblocks to let EasyBuild framework take care of prepend to $PYTHONPATH or $EBPYTHONPREFIXES + make PythonBundle and PythonPackage aware of --prefer-python-search-path EasyBuild configuration option

easybuild/easyblocks/a/amber.py

@@ -381,7 +381,5 @@ def make_module_extra(self):
         txt = super(EB_Amber, self).make_module_extra()
 
         txt += self.module_generator.set_environment('AMBERHOME', self.installdir)
-        if self.pylibdir:
-            txt += self.module_generator.prepend_paths('PYTHONPATH', self.pylibdir)
 
         return txt

easybuild/easyblocks/c/cplex.py

@@ -38,8 +38,6 @@
 
 import easybuild.tools.environment as env
 from easybuild.easyblocks.generic.binary import Binary
-from easybuild.easyblocks.generic.pythonpackage import det_pylibdir
-from easybuild.easyblocks.python import EBPYTHONPREFIXES
 from easybuild.tools.build_log import EasyBuildError
 from easybuild.tools.filetools import adjust_permissions, change_dir, mkdir
 from easybuild.tools.modules import get_software_root
@@ -150,12 +148,6 @@ def make_module_extra(self):
         txt += self.module_generator.set_environment('CPLEX_HOME', os.path.join(self.installdir, 'cplex'))
         txt += self.module_generator.set_environment('CPLEXDIR', os.path.join(self.installdir, 'cplex'))
 
-        if self.with_python:
-            if self.multi_python:
-                txt += self.module_generator.prepend_paths(EBPYTHONPREFIXES, '')
-            else:
-                txt += self.module_generator.prepend_paths('PYTHONPATH', [det_pylibdir()])
-
         self.log.debug("make_module_extra added %s" % txt)
         return txt
 

easybuild/easyblocks/e/esmf.py

@@ -148,23 +148,14 @@ def install_step(self):
             run_shell_cmd(cmd)
 
     def make_module_extra(self):
-        """Add install path to PYTHONPATH or EBPYTHONPREFIXES"""
+        """Set $ESMFMKFILE environment variable"""
         txt = super(EB_ESMF, self).make_module_extra()
 
         # set environment variable ESMFMKFILE
         # see section 9.9 in https://earthsystemmodeling.org/docs/release/latest/ESMF_usrdoc/node10.html
         esmf_mkfile_path = os.path.join(self.installdir, "lib", "esmf.mk")
         txt += self.module_generator.set_environment('ESMFMKFILE', esmf_mkfile_path)
 
-        if self.cfg['multi_deps'] and 'Python' in self.cfg['multi_deps']:
-            txt += self.module_generator.prepend_paths('EBPYTHONPREFIXES', '')
-        else:
-            python = get_software_version('Python')
-            if python:
-                pyshortver = '.'.join(get_software_version('Python').split('.')[:2])
-                pythonpath = os.path.join('lib', 'python%s' % pyshortver, 'site-packages')
-                txt += self.module_generator.prepend_paths('PYTHONPATH', [pythonpath])
-
         return txt
 
     def sanity_check_step(self):

easybuild/easyblocks/g/gurobi.py

@@ -31,7 +31,6 @@
 """
 import os
 
-from easybuild.easyblocks.generic.pythonpackage import det_pylibdir
 from easybuild.easyblocks.generic.tarball import Tarball
 from easybuild.framework.easyconfig import CUSTOM
 from easybuild.tools import LooseVersion
@@ -101,10 +100,6 @@ def make_module_extra(self):
         txt = super(EB_Gurobi, self).make_module_extra()
         txt += self.module_generator.set_environment('GUROBI_HOME', self.installdir)
         txt += self.module_generator.set_environment('GRB_LICENSE_FILE', self.license_file)
-
-        if get_software_root('Python'):
-            txt += self.module_generator.prepend_paths('PYTHONPATH', det_pylibdir())
-
         txt += self.module_generator.prepend_paths('MATLABPATH', 'matlab')
 
         return txt

easybuild/easyblocks/generic/pythonbundle.py

@@ -31,9 +31,10 @@
 import sys
 
 from easybuild.easyblocks.generic.bundle import Bundle
-from easybuild.easyblocks.generic.pythonpackage import EBPYTHONPREFIXES, EXTS_FILTER_PYTHON_PACKAGES
+from easybuild.easyblocks.generic.pythonpackage import EXTS_FILTER_PYTHON_PACKAGES
 from easybuild.easyblocks.generic.pythonpackage import PythonPackage, get_pylibdirs, pick_python_cmd
 from easybuild.tools.build_log import EasyBuildError
+from easybuild.tools.config import build_option, PYTHONPATH, EBPYTHONPREFIXES
 from easybuild.tools.filetools import which
 from easybuild.tools.modules import get_software_root
 import easybuild.tools.environment as env
@@ -146,9 +147,20 @@ def make_module_extra(self, *args, **kwargs):
         txt = super(Bundle, self).make_module_extra(*args, **kwargs)
 
         # update $EBPYTHONPREFIXES rather than $PYTHONPATH
-        # if this Python package was installed for multiple Python versions
-        if self.multi_python:
-            txt += self.module_generator.prepend_paths(EBPYTHONPREFIXES, '')
+        # if this Python package was installed for multiple Python versions, or if we prefer it
+        use_ebpythonprefixes = False
+        runtime_deps = [dep['name'] for dep in self.cfg.dependencies(runtime_only=True)]
+
+        if 'Python' in runtime_deps:
+            self.log.info("Found Python runtime dependency, so considering $EBPYTHONPREFIXES...")
+            if build_option('prefer_python_search_path') == EBPYTHONPREFIXES:
+                self.log.info("Preferred Python search path is $EBPYTHONPREFIXES, so using that")
+                use_ebpythonprefixes = True
+
+        if self.multi_python or use_ebpythonprefixes:
+            path = ''  # EBPYTHONPREFIXES are relative to the install dir
+            if path not in self.module_generator.added_paths_per_key[EBPYTHONPREFIXES]:
+                txt += self.module_generator.prepend_paths(EBPYTHONPREFIXES, path)
         else:
 
             # the temporary module file that is generated before installing extensions
@@ -163,7 +175,8 @@ def make_module_extra(self, *args, **kwargs):
                 ]
 
             for pylibdir in new_pylibdirs:
-                txt += self.module_generator.prepend_paths('PYTHONPATH', pylibdir)
+                if pylibdir not in self.module_generator.added_paths_per_key[PYTHONPATH]:
+                    txt += self.module_generator.prepend_paths(PYTHONPATH, pylibdir)
 
         return txt
 

easybuild/easyblocks/generic/pythonpackage.py

@@ -42,13 +42,13 @@
 
 import easybuild.tools.environment as env
 from easybuild.base import fancylogger
-from easybuild.easyblocks.python import EBPYTHONPREFIXES, EXTS_FILTER_PYTHON_PACKAGES
+from easybuild.easyblocks.python import EXTS_FILTER_PYTHON_PACKAGES
 from easybuild.framework.easyconfig import CUSTOM
 from easybuild.framework.easyconfig.default import DEFAULT_CONFIG
 from easybuild.framework.easyconfig.templates import PYPI_SOURCE
 from easybuild.framework.extensioneasyblock import ExtensionEasyBlock
 from easybuild.tools.build_log import EasyBuildError, print_msg
-from easybuild.tools.config import build_option
+from easybuild.tools.config import build_option, PYTHONPATH, EBPYTHONPREFIXES
 from easybuild.tools.filetools import change_dir, mkdir, remove_dir, symlink, which
 from easybuild.tools.modules import get_software_root
 from easybuild.tools.run import run_shell_cmd, subprocess_popen_text
@@ -1162,15 +1162,27 @@ def make_module_extra(self, *args, **kwargs):
         txt = ''
 
         # update $EBPYTHONPREFIXES rather than $PYTHONPATH
-        # if this Python package was installed for multiple Python versions
-        if self.multi_python:
-            txt += self.module_generator.prepend_paths(EBPYTHONPREFIXES, '')
+        # if this Python package was installed for multiple Python versions, or if we prefer it;
+        # note: although EasyBuild framework also has logic for this in EasyBlock.make_module_extra,
+        # we retain full control here, since the logic is slightly different
+        use_ebpythonprefixes = False
+        runtime_deps = [dep['name'] for dep in self.cfg.dependencies(runtime_only=True)]
+
+        if 'Python' in runtime_deps:
+            self.log.info("Found Python runtime dependency, so considering $EBPYTHONPREFIXES...")
+            if build_option('prefer_python_search_path') == EBPYTHONPREFIXES:
+                self.log.info("Preferred Python search path is $EBPYTHONPREFIXES, so using that")
+                use_ebpythonprefixes = True
+
+        if self.multi_python or use_ebpythonprefixes:
+            path = ''  # EBPYTHONPREFIXES are relative to the install dir
+            txt += self.module_generator.prepend_paths(EBPYTHONPREFIXES, path)
         elif self.require_python:
             self.set_pylibdirs()
             for path in self.all_pylibdirs:
                 fullpath = os.path.join(self.installdir, path)
                 # only extend $PYTHONPATH with existing, non-empty directories
                 if os.path.exists(fullpath) and os.listdir(fullpath):
-                    txt += self.module_generator.prepend_paths('PYTHONPATH', path)
+                    txt += self.module_generator.prepend_paths(PYTHONPATH, path)
 
         return super(PythonPackage, self).make_module_extra(txt, *args, **kwargs)

easybuild/easyblocks/l/lammps.py

@@ -548,19 +548,6 @@ def sanity_check_step(self, *args, **kwargs):
 
         return super(EB_LAMMPS, self).sanity_check_step(custom_commands=custom_commands, custom_paths=custom_paths)
 
-    def make_module_extra(self):
-        """Add install path to PYTHONPATH"""
-
-        txt = super(EB_LAMMPS, self).make_module_extra()
-
-        python = get_software_version('Python')
-        if python:
-            pyshortver = '.'.join(get_software_version('Python').split('.')[:2])
-            pythonpath = os.path.join('lib', 'python%s' % pyshortver, 'site-packages')
-            txt += self.module_generator.prepend_paths('PYTHONPATH', [pythonpath])
-
-        return txt
-
 
 def get_cuda_gpu_arch(cuda_cc):
     """Return CUDA gpu ARCH in LAMMPS required format. Example: 'sm_32' """

easybuild/easyblocks/m/mxnet.py

@@ -252,12 +252,6 @@ def make_module_extra(self, *args, **kwargs):
         """Custom variables for MXNet module."""
         txt = super(EB_MXNet, self).make_module_extra(*args, **kwargs)
 
-        for path in self.py_ext.all_pylibdirs:
-            fullpath = os.path.join(self.installdir, path)
-            # only extend $PYTHONPATH with existing, non-empty directories
-            if os.path.exists(fullpath) and os.listdir(fullpath):
-                txt += self.module_generator.prepend_paths('PYTHONPATH', path)
-
         txt += self.module_generator.prepend_paths("R_LIBS", [''])  # prepend R_LIBS with install path
 
         return txt

easybuild/easyblocks/n/neuron.py

@@ -283,11 +283,4 @@ def make_module_extra(self):
             else:
                 self.log.debug("%s not set: %s" % (var, os.environ.get(var, None)))
 
-        if self.with_python:
-            if self.cfg['multi_deps'] and 'Python' in self.cfg['multi_deps']:
-                txt += self.module_generator.prepend_paths('EBPYTHONPREFIXES', '')
-            else:
-                txt += self.module_generator.prepend_paths('PYTHONPATH', [self.pylibdir])
-            # also adds lib/python to PYTHONPATH
-            txt += self.module_generator.prepend_paths('PYTHONPATH', ['lib/python'])
         return txt

easybuild/easyblocks/o/openbabel.py

@@ -31,7 +31,6 @@
 import glob
 import os
 from easybuild.easyblocks.generic.cmakemake import CMakeMake
-from easybuild.easyblocks.generic.pythonpackage import det_pylibdir
 from easybuild.framework.easyconfig import CUSTOM
 from easybuild.tools.build_log import EasyBuildError
 from easybuild.tools.modules import get_software_root, get_software_version
@@ -108,14 +107,6 @@ def sanity_check_step(self):
     def make_module_extra(self):
         """Custom variables for OpenBabel module."""
         txt = super(EB_OpenBabel, self).make_module_extra()
-        if self.with_python:
-            if LooseVersion(self.version) >= LooseVersion('2.4'):
-                # since OpenBabel 2.4.0 the Python bindings under
-                # ${PREFIX}/lib/python2.7/site-packages  rather than ${PREFIX}/lib
-                ob_pythonpath = det_pylibdir()
-            else:
-                ob_pythonpath = 'lib'
-            txt += self.module_generator.prepend_paths('PYTHONPATH', [ob_pythonpath])
         babel_libdir = os.path.join(self.installdir, 'lib', 'openbabel', self.version)
         txt += self.module_generator.set_environment('BABEL_LIBDIR', babel_libdir)
         babel_datadir = os.path.join(self.installdir, 'share', 'openbabel', self.version)

easybuild/easyblocks/o/opencv.py

@@ -248,7 +248,4 @@ def make_module_extra(self):
 
         txt += self.module_generator.prepend_paths('CLASSPATH', os.path.join('share', 'OpenCV', 'java'))
 
-        if os.path.exists(os.path.join(self.installdir, self.pylibdir)):
-            txt += self.module_generator.prepend_paths('PYTHONPATH', self.pylibdir)
-
         return txt

easybuild/easyblocks/p/python.py

@@ -45,7 +45,7 @@
 from easybuild.framework.easyconfig import CUSTOM
 from easybuild.framework.easyconfig.templates import PYPI_SOURCE
 from easybuild.tools.build_log import EasyBuildError, print_warning
-from easybuild.tools.config import build_option, ERROR, log_path
+from easybuild.tools.config import build_option, ERROR, log_path, PYTHONPATH, EBPYTHONPREFIXES
 from easybuild.tools.modules import get_software_libdir, get_software_root, get_software_version
 from easybuild.tools.filetools import apply_regex_substitutions, change_dir, mkdir
 from easybuild.tools.filetools import read_file, remove_dir, symlink, write_file
@@ -59,8 +59,6 @@
 # magic value for unlimited stack size
 UNLIMITED = 'unlimited'
 
-EBPYTHONPREFIXES = 'EBPYTHONPREFIXES'
-
 # We want the following import order:
 # 1. Packages installed into VirtualEnv
 # 2. Packages installed into $EBPYTHONPREFIXES (e.g. our modules)
@@ -643,6 +641,6 @@ def make_module_extra(self, *args, **kwargs):
         txt = super(EB_Python, self).make_module_extra()
 
         if self.cfg.get('ebpythonprefixes'):
-            txt += self.module_generator.prepend_paths('PYTHONPATH', self.pythonpath)
+            txt += self.module_generator.prepend_paths(PYTHONPATH, self.pythonpath)
 
         return txt

easybuild/easyblocks/q/qscintilla.py

@@ -200,14 +200,3 @@ def sanity_check_step(self):
             custom_commands.append("python -c 'import %s.Qsci'" % self.pyqt_pkg_name)
 
         super(EB_QScintilla, self).sanity_check_step(custom_paths=custom_paths, custom_commands=custom_commands)
-
-    def make_module_extra(self):
-        """Custom extra module file entries for QScintilla."""
-        txt = super(EB_QScintilla, self).make_module_extra()
-        python = get_software_root('Python')
-        if python:
-            if self.cfg['multi_deps'] and 'Python' in self.cfg['multi_deps']:
-                txt += self.module_generator.prepend_paths('EBPYTHONPREFIXES', '')
-            else:
-                txt += self.module_generator.prepend_paths('PYTHONPATH', [det_pylibdir()])
-        return txt

easybuild/easyblocks/t/tkinter.py

@@ -134,10 +134,3 @@ def sanity_check_step(self):
             'dirs': ['lib']
         }
         super(EB_Python, self).sanity_check_step(custom_commands=custom_commands, custom_paths=custom_paths)
-
-    def make_module_extra(self):
-        """Set PYTHONPATH"""
-        txt = super(EB_Tkinter, self).make_module_extra()
-        txt += self.module_generator.prepend_paths('PYTHONPATH', det_pylibdir())
-
-        return txt