enhance PythonPackage and PythonBundle easyblocks to add support to install dummy Python packages

easybuild/easyblocks/generic/pythonbundle.py

@@ -30,8 +30,9 @@
 import os
 
 from easybuild.easyblocks.generic.bundle import Bundle
-from easybuild.easyblocks.generic.pythonpackage import EXTS_FILTER_PYTHON_PACKAGES, run_pip_check, set_py_env_vars
+from easybuild.easyblocks.generic.pythonpackage import EXTS_FILTER_DUMMY_PACKAGES, EXTS_FILTER_PYTHON_PACKAGES
 from easybuild.easyblocks.generic.pythonpackage import PythonPackage, get_pylibdirs, find_python_cmd_from_ec
+from easybuild.easyblocks.generic.pythonpackage import run_pip_check, set_py_env_vars
 from easybuild.tools.build_log import EasyBuildError
 from easybuild.tools.config import build_option, PYTHONPATH, EBPYTHONPREFIXES
 from easybuild.tools.modules import get_software_root
@@ -60,6 +61,8 @@ def __init__(self, *args, **kwargs):
 
         self.cfg['exts_defaultclass'] = 'PythonPackage'
         self.cfg['exts_filter'] = EXTS_FILTER_PYTHON_PACKAGES
+        if self.cfg.get('dummy_package', False):
+            self.cfg['exts_filter'] = EXTS_FILTER_DUMMY_PACKAGES
 
         # need to disable templating to ensure that actual value for exts_default_options is updated...
         with self.cfg.disable_templating():
@@ -71,6 +74,12 @@ def __init__(self, *args, **kwargs):
 
             self.log.info("exts_default_options: %s", self.cfg['exts_default_options'])
 
+            # dummy packages have no sources sources
+            if self.cfg.get('dummy_package', False):
+                self.log.info(f"Disabling sources for installation of dummy packages in {self.name}-{self.version}")
+                self.cfg['exts_default_options']['nosource'] = True
+                self.cfg['exts_default_options']['source_urls'] = []
+
         self.python_cmd = None
         self.pylibdir = None
         self.all_pylibdirs = None
@@ -91,8 +100,9 @@ def prepare_python(self):
         # if 'python' is not used, we need to take that into account in the extensions filter
         # (which is also used during the sanity check)
         if self.python_cmd != 'python':
-            orig_exts_filter = EXTS_FILTER_PYTHON_PACKAGES
-            self.cfg['exts_filter'] = (orig_exts_filter[0].replace('python', self.python_cmd), orig_exts_filter[1])
+            with self.cfg.disable_templating():
+                orig_exts_filter = self.cfg['exts_filter']
+                self.cfg['exts_filter'] = (orig_exts_filter[0].replace('python', self.python_cmd), orig_exts_filter[1])
 
     def prepare_step(self, *args, **kwargs):
         """Prepare for installing bundle of Python packages."""

easybuild/easyblocks/generic/pythonpackage.py

@@ -41,7 +41,7 @@
 
 import easybuild.tools.environment as env
 from easybuild.base import fancylogger
-from easybuild.easyblocks.python import EXTS_FILTER_PYTHON_PACKAGES, set_py_env_vars
+from easybuild.easyblocks.python import EXTS_FILTER_DUMMY_PACKAGES, EXTS_FILTER_PYTHON_PACKAGES, set_py_env_vars
 from easybuild.easyblocks.python import det_installed_python_packages, det_pip_version, run_pip_check
 from easybuild.framework.easyconfig import CUSTOM
 from easybuild.framework.easyconfig.default import DEFAULT_CONFIG
@@ -488,6 +488,8 @@ def extra_options(extra_vars=None):
             'download_dep_fail': [None, "Fail if downloaded dependencies are detected. "
                                   "Defaults to True unless 'use_pip_for_deps' or 'use_pip_requirement' is True.",
                                   CUSTOM],
+            'dummy_package': [None, "Install a dummy package empty in contents but visible by Python package managers "
+                                    "such as pip", CUSTOM],
             'fix_python_shebang_for': [['bin/*'], "List of files for which Python shebang should be fixed "
                                                   "to '#!/usr/bin/env python' (glob patterns supported) "
                                                   "(default: ['bin/*'])", CUSTOM],
@@ -560,6 +562,12 @@ def __init__(self, *args, **kwargs):
         if os.path.exists(os.path.join(home, 'site.cfg')):
             raise EasyBuildError("Found site.cfg in your home directory (%s), please remove it.", home)
 
+        # dummy packages have no sources
+        if self.cfg.get('dummy_package', False):
+            self.log.info(f"Disabling sources for dummy package {self.name}-{self.version}")
+            self.cfg['source_urls'] = []
+            self.cfg['sources'] = []
+
         # use lowercase name as default value for expected module name (used in sanity check)
         if 'modulename' not in self.options:
             self.options['modulename'] = self.name.lower().replace('-', '_')
@@ -832,6 +840,28 @@ def compose_install_command(self, prefix, extrapath=None, installopts=None, inst
 
         return ' '.join(cmd)
 
+    def install_dummy_package(self):
+        """
+        Create dist-info directory inside site-packages with the metadata for
+        the given target package
+        """
+        py_package_metadata = [
+            "Metadata-Version: 2.1",
+            f"Name: {self.name}",
+            f"Version: {self.version}",
+        ]
+
+        # make dist-info directory
+        dist_info_name = self.name.replace('-', '_') + f"-{self.version}.dist-info"
+        dist_info_path = os.path.join(self.installdir, self.pylibdir, dist_info_name)
+        mkdir(dist_info_path, parents=True)
+
+        # install METADATA file
+        metadata_path = os.path.join(dist_info_path, 'METADATA')
+        write_file(metadata_path, '\n'.join(py_package_metadata))
+
+        self.log.info(f"Installation of dummy package for {self.name}-{self.version} successfull: {metadata_path}")
+
     def py_post_install_shenanigans(self, install_dir):
         """
         Run post-installation shenanigans on specified installation directory, incl:
@@ -929,6 +959,10 @@ def configure_step(self):
     def build_step(self):
         """Build Python package using setup.py"""
 
+        if self.cfg.get('dummy_package', False):
+            self.log.info(f"Skipping build step for installation of dummy package {self.name}-{self.version}")
+            return
+
         # inject extra '%(python)s' template value before getting value of 'buildcmd' custom easyconfig parameter
         self.cfg.template_values['python'] = self.python_cmd
         build_cmd = self.cfg['buildcmd']
@@ -961,6 +995,10 @@ def test_step(self, return_output_ec=False):
         :param return_output: return output and exit code of test command
         """
 
+        if self.cfg.get('dummy_package', False):
+            self.log.info(f"Skipping test step for installation of dummy package {self.name}-{self.version}")
+            return None
+
         if isinstance(self.cfg['runtest'], str):
             self.testcmd = self.cfg['runtest']
 
@@ -1032,9 +1070,15 @@ def test_step(self, return_output_ec=False):
             if return_output_ec:
                 return (out, ec)
 
+        return None
+
     def install_step(self):
         """Install Python package to a custom path using setup.py"""
 
+        if self.cfg.get('dummy_package', False):
+            self.install_dummy_package()
+            return
+
         # if posix_local is the active installation scheme there will be
         # a 'local' subdirectory in the specified prefix;
         # see also https://github.com/easybuilders/easybuild-easyblocks/issues/2976
@@ -1083,6 +1127,10 @@ def install_step(self):
     def install_extension(self, *args, **kwargs):
         """Perform the actual Python package build/installation procedure"""
 
+        if self.cfg.get('dummy_package', False):
+            self.install_dummy_package()
+            return
+
         # we unpack unless explicitly told otherwise
         kwargs.setdefault('unpack_src', self._should_unpack_source())
         super().install_extension(*args, **kwargs)
@@ -1173,28 +1221,40 @@ def sanity_check_step(self, *args, **kwargs):
         # this is relevant for installations of Python packages for multiple Python versions (via multi_deps)
         # (we can not pass this via custom_paths, since then the %(pyshortver)s template value will not be resolved)
         if not self.is_extension:
-            kwargs.setdefault('custom_paths', {'files': []}) \
-                  .setdefault('dirs', [os.path.join('lib', 'python%(pyshortver)s', 'site-packages')])
+            site_package_dir = os.path.join('lib', 'python%(pyshortver)s', 'site-packages')
+
+            custom_paths_files = []
+            if self.cfg.get('dummy_package', False):
+                dist_info_name = self.name.replace('-', '_') + f'-{self.version}.dist-info'
+                custom_paths_files.append(os.path.join(site_package_dir, dist_info_name, 'METADATA'))
+
+            kwargs.setdefault('custom_paths', {
+                'files': custom_paths_files,
+                'dirs': [site_package_dir],
+            })
+
+        # make sure 'exts_filter' argument is defined, which is used for sanity check
+        exts_sanity_filter = EXTS_FILTER_PYTHON_PACKAGES
+        if self.cfg.get('dummy_package', False):
+            exts_sanity_filter = EXTS_FILTER_DUMMY_PACKAGES
 
-        # make sure 'exts_filter' is defined, which is used for sanity check
         if self.multi_python:
             # when installing for multiple Python versions, we must use 'python', not a full-path 'python' command!
-            python_cmd = 'python'
+            pip_check_python_cmd = 'python'
             if 'exts_filter' not in kwargs:
-                kwargs.update({'exts_filter': EXTS_FILTER_PYTHON_PACKAGES})
+                kwargs.update({'exts_filter': exts_sanity_filter})
         else:
             # 'python' is replaced by full path to active 'python' command
             # (which is required especially when installing with system Python)
             if self.python_cmd is None:
                 self.prepare_python()
-            python_cmd = self.python_cmd
+            pip_check_python_cmd = self.python_cmd
             if 'exts_filter' not in kwargs:
-                orig_exts_filter = EXTS_FILTER_PYTHON_PACKAGES
-                exts_filter = (orig_exts_filter[0].replace('python', self.python_cmd), orig_exts_filter[1])
+                exts_filter = (exts_sanity_filter[0].replace('python', self.python_cmd), exts_sanity_filter[1])
                 kwargs.update({'exts_filter': exts_filter})
 
         # inject extra '%(python)s' template value for use by sanity check commands
-        self.cfg.template_values['python'] = python_cmd
+        self.cfg.template_values['python'] = pip_check_python_cmd
 
         sanity_pip_check = self.cfg.get('sanity_pip_check', True)
         if self.is_extension:
@@ -1220,7 +1280,7 @@ def sanity_check_step(self, *args, **kwargs):
                                          self.short_mod_name)
 
             unversioned_packages = self.cfg.get('unversioned_packages', [])
-            run_pip_check(python_cmd=python_cmd, unversioned_packages=unversioned_packages)
+            run_pip_check(python_cmd=pip_check_python_cmd, unversioned_packages=unversioned_packages)
 
         # ExtensionEasyBlock handles loading modules correctly for multi_deps, so we clean up fake_mod_data
         # and let ExtensionEasyBlock do its job

easybuild/easyblocks/p/python.py

@@ -58,6 +58,7 @@
 
 
 EXTS_FILTER_PYTHON_PACKAGES = ('python -c "import %(ext_name)s"', "")
+EXTS_FILTER_DUMMY_PACKAGES = ("python -m pip show -q '%(ext_name)s'", "")
 
 # magic value for unlimited stack size
 UNLIMITED = 'unlimited'