diff --git a/easybuild/easyblocks/generic/pythonbundle.py b/easybuild/easyblocks/generic/pythonbundle.py index e678c432f42..ae66d3b21c7 100644 --- a/easybuild/easyblocks/generic/pythonbundle.py +++ b/easybuild/easyblocks/generic/pythonbundle.py @@ -30,8 +30,9 @@ import os from easybuild.easyblocks.generic.bundle import Bundle -from easybuild.easyblocks.generic.pythonpackage import EXTS_FILTER_PYTHON_PACKAGES, run_pip_check, set_py_env_vars +from easybuild.easyblocks.generic.pythonpackage import EXTS_FILTER_DUMMY_PACKAGES, EXTS_FILTER_PYTHON_PACKAGES from easybuild.easyblocks.generic.pythonpackage import PythonPackage, get_pylibdirs, find_python_cmd_from_ec +from easybuild.easyblocks.generic.pythonpackage import run_pip_check, set_py_env_vars from easybuild.tools.build_log import EasyBuildError from easybuild.tools.config import build_option, PYTHONPATH, EBPYTHONPREFIXES from easybuild.tools.modules import get_software_root @@ -60,6 +61,8 @@ def __init__(self, *args, **kwargs): self.cfg['exts_defaultclass'] = 'PythonPackage' self.cfg['exts_filter'] = EXTS_FILTER_PYTHON_PACKAGES + if self.cfg.get('dummy_package', False): + self.cfg['exts_filter'] = EXTS_FILTER_DUMMY_PACKAGES # need to disable templating to ensure that actual value for exts_default_options is updated... with self.cfg.disable_templating(): @@ -71,6 +74,12 @@ def __init__(self, *args, **kwargs): self.log.info("exts_default_options: %s", self.cfg['exts_default_options']) + # dummy packages have no sources sources + if self.cfg.get('dummy_package', False): + self.log.info(f"Disabling sources for installation of dummy packages in {self.name}-{self.version}") + self.cfg['exts_default_options']['nosource'] = True + self.cfg['exts_default_options']['source_urls'] = [] + self.python_cmd = None self.pylibdir = None self.all_pylibdirs = None @@ -91,8 +100,9 @@ def prepare_python(self): # if 'python' is not used, we need to take that into account in the extensions filter # (which is also used during the sanity check) if self.python_cmd != 'python': - orig_exts_filter = EXTS_FILTER_PYTHON_PACKAGES - self.cfg['exts_filter'] = (orig_exts_filter[0].replace('python', self.python_cmd), orig_exts_filter[1]) + with self.cfg.disable_templating(): + orig_exts_filter = self.cfg['exts_filter'] + self.cfg['exts_filter'] = (orig_exts_filter[0].replace('python', self.python_cmd), orig_exts_filter[1]) def prepare_step(self, *args, **kwargs): """Prepare for installing bundle of Python packages.""" diff --git a/easybuild/easyblocks/generic/pythonpackage.py b/easybuild/easyblocks/generic/pythonpackage.py index 14f12856fd1..59b4d87dbbf 100644 --- a/easybuild/easyblocks/generic/pythonpackage.py +++ b/easybuild/easyblocks/generic/pythonpackage.py @@ -41,7 +41,7 @@ import easybuild.tools.environment as env from easybuild.base import fancylogger -from easybuild.easyblocks.python import EXTS_FILTER_PYTHON_PACKAGES, set_py_env_vars +from easybuild.easyblocks.python import EXTS_FILTER_DUMMY_PACKAGES, EXTS_FILTER_PYTHON_PACKAGES, set_py_env_vars from easybuild.easyblocks.python import det_installed_python_packages, det_pip_version, run_pip_check from easybuild.framework.easyconfig import CUSTOM from easybuild.framework.easyconfig.default import DEFAULT_CONFIG @@ -488,6 +488,8 @@ def extra_options(extra_vars=None): 'download_dep_fail': [None, "Fail if downloaded dependencies are detected. " "Defaults to True unless 'use_pip_for_deps' or 'use_pip_requirement' is True.", CUSTOM], + 'dummy_package': [None, "Install a dummy package empty in contents but visible by Python package managers " + "such as pip", CUSTOM], 'fix_python_shebang_for': [['bin/*'], "List of files for which Python shebang should be fixed " "to '#!/usr/bin/env python' (glob patterns supported) " "(default: ['bin/*'])", CUSTOM], @@ -560,6 +562,12 @@ def __init__(self, *args, **kwargs): if os.path.exists(os.path.join(home, 'site.cfg')): raise EasyBuildError("Found site.cfg in your home directory (%s), please remove it.", home) + # dummy packages have no sources + if self.cfg.get('dummy_package', False): + self.log.info(f"Disabling sources for dummy package {self.name}-{self.version}") + self.cfg['source_urls'] = [] + self.cfg['sources'] = [] + # use lowercase name as default value for expected module name (used in sanity check) if 'modulename' not in self.options: self.options['modulename'] = self.name.lower().replace('-', '_') @@ -832,6 +840,28 @@ def compose_install_command(self, prefix, extrapath=None, installopts=None, inst return ' '.join(cmd) + def install_dummy_package(self): + """ + Create dist-info directory inside site-packages with the metadata for + the given target package + """ + py_package_metadata = [ + "Metadata-Version: 2.1", + f"Name: {self.name}", + f"Version: {self.version}", + ] + + # make dist-info directory + dist_info_name = self.name.replace('-', '_') + f"-{self.version}.dist-info" + dist_info_path = os.path.join(self.installdir, self.pylibdir, dist_info_name) + mkdir(dist_info_path, parents=True) + + # install METADATA file + metadata_path = os.path.join(dist_info_path, 'METADATA') + write_file(metadata_path, '\n'.join(py_package_metadata)) + + self.log.info(f"Installation of dummy package for {self.name}-{self.version} successfull: {metadata_path}") + def py_post_install_shenanigans(self, install_dir): """ Run post-installation shenanigans on specified installation directory, incl: @@ -929,6 +959,10 @@ def configure_step(self): def build_step(self): """Build Python package using setup.py""" + if self.cfg.get('dummy_package', False): + self.log.info(f"Skipping build step for installation of dummy package {self.name}-{self.version}") + return + # inject extra '%(python)s' template value before getting value of 'buildcmd' custom easyconfig parameter self.cfg.template_values['python'] = self.python_cmd build_cmd = self.cfg['buildcmd'] @@ -961,6 +995,10 @@ def test_step(self, return_output_ec=False): :param return_output: return output and exit code of test command """ + if self.cfg.get('dummy_package', False): + self.log.info(f"Skipping test step for installation of dummy package {self.name}-{self.version}") + return None + if isinstance(self.cfg['runtest'], str): self.testcmd = self.cfg['runtest'] @@ -1032,9 +1070,15 @@ def test_step(self, return_output_ec=False): if return_output_ec: return (out, ec) + return None + def install_step(self): """Install Python package to a custom path using setup.py""" + if self.cfg.get('dummy_package', False): + self.install_dummy_package() + return + # if posix_local is the active installation scheme there will be # a 'local' subdirectory in the specified prefix; # see also https://github.com/easybuilders/easybuild-easyblocks/issues/2976 @@ -1083,6 +1127,10 @@ def install_step(self): def install_extension(self, *args, **kwargs): """Perform the actual Python package build/installation procedure""" + if self.cfg.get('dummy_package', False): + self.install_dummy_package() + return + # we unpack unless explicitly told otherwise kwargs.setdefault('unpack_src', self._should_unpack_source()) super().install_extension(*args, **kwargs) @@ -1173,28 +1221,40 @@ def sanity_check_step(self, *args, **kwargs): # this is relevant for installations of Python packages for multiple Python versions (via multi_deps) # (we can not pass this via custom_paths, since then the %(pyshortver)s template value will not be resolved) if not self.is_extension: - kwargs.setdefault('custom_paths', {'files': []}) \ - .setdefault('dirs', [os.path.join('lib', 'python%(pyshortver)s', 'site-packages')]) + site_package_dir = os.path.join('lib', 'python%(pyshortver)s', 'site-packages') + + custom_paths_files = [] + if self.cfg.get('dummy_package', False): + dist_info_name = self.name.replace('-', '_') + f'-{self.version}.dist-info' + custom_paths_files.append(os.path.join(site_package_dir, dist_info_name, 'METADATA')) + + kwargs.setdefault('custom_paths', { + 'files': custom_paths_files, + 'dirs': [site_package_dir], + }) + + # make sure 'exts_filter' argument is defined, which is used for sanity check + exts_sanity_filter = EXTS_FILTER_PYTHON_PACKAGES + if self.cfg.get('dummy_package', False): + exts_sanity_filter = EXTS_FILTER_DUMMY_PACKAGES - # make sure 'exts_filter' is defined, which is used for sanity check if self.multi_python: # when installing for multiple Python versions, we must use 'python', not a full-path 'python' command! - python_cmd = 'python' + pip_check_python_cmd = 'python' if 'exts_filter' not in kwargs: - kwargs.update({'exts_filter': EXTS_FILTER_PYTHON_PACKAGES}) + kwargs.update({'exts_filter': exts_sanity_filter}) else: # 'python' is replaced by full path to active 'python' command # (which is required especially when installing with system Python) if self.python_cmd is None: self.prepare_python() - python_cmd = self.python_cmd + pip_check_python_cmd = self.python_cmd if 'exts_filter' not in kwargs: - orig_exts_filter = EXTS_FILTER_PYTHON_PACKAGES - exts_filter = (orig_exts_filter[0].replace('python', self.python_cmd), orig_exts_filter[1]) + exts_filter = (exts_sanity_filter[0].replace('python', self.python_cmd), exts_sanity_filter[1]) kwargs.update({'exts_filter': exts_filter}) # inject extra '%(python)s' template value for use by sanity check commands - self.cfg.template_values['python'] = python_cmd + self.cfg.template_values['python'] = pip_check_python_cmd sanity_pip_check = self.cfg.get('sanity_pip_check', True) if self.is_extension: @@ -1220,7 +1280,7 @@ def sanity_check_step(self, *args, **kwargs): self.short_mod_name) unversioned_packages = self.cfg.get('unversioned_packages', []) - run_pip_check(python_cmd=python_cmd, unversioned_packages=unversioned_packages) + run_pip_check(python_cmd=pip_check_python_cmd, unversioned_packages=unversioned_packages) # ExtensionEasyBlock handles loading modules correctly for multi_deps, so we clean up fake_mod_data # and let ExtensionEasyBlock do its job diff --git a/easybuild/easyblocks/p/python.py b/easybuild/easyblocks/p/python.py index 32b69db001c..2fa7806b876 100644 --- a/easybuild/easyblocks/p/python.py +++ b/easybuild/easyblocks/p/python.py @@ -58,6 +58,7 @@ EXTS_FILTER_PYTHON_PACKAGES = ('python -c "import %(ext_name)s"', "") +EXTS_FILTER_DUMMY_PACKAGES = ("python -m pip show -q '%(ext_name)s'", "") # magic value for unlimited stack size UNLIMITED = 'unlimited'