diff --git a/easybuild/easyblocks/p/python.py b/easybuild/easyblocks/p/python.py index d0ef2d40f1f..bc00c86b03a 100644 --- a/easybuild/easyblocks/p/python.py +++ b/easybuild/easyblocks/p/python.py @@ -302,6 +302,13 @@ def extra_options(): 'ulimit_unlimited': [False, "Ensure stack size limit is set to '%s' during build" % UNLIMITED, CUSTOM], 'use_lto': [None, "Build with Link Time Optimization (>= v3.7.0, potentially unstable on some toolchains). " "If None: auto-detect based on toolchain compiler (version)", CUSTOM], + 'patch_ctypes_ld_library_path': [None, + "The ctypes module strongly relies on LD_LIBRARY_PATH to find " + "libraries. This allows specifying a patch that will only be " + "applied if EasyBuild is configured to filter LD_LIBRARY_PATH, in " + "order to make sure ctypes can still find libraries without it. " + "Please make sure to add the checksum for this patch to 'checksums'.", + CUSTOM], } return ConfigureMake.extra_options(extra_vars) @@ -345,6 +352,64 @@ def _get_pip_ext_version(self): return ext[1] return None + def fetch_step(self, *args, **kwargs): + """ + Custom fetch step for Python. + + Add patch specified in patch_ctypes_ld_library_path to list of patches if + EasyBuild is configured to filter $LD_LIBRARY_PATH (and is configured not to filter $LIBRARY_PATH). + This needs to be done in (or before) the fetch step to ensure that those patches are also fetched. + """ + # If we filter out $LD_LIBRARY_PATH (not unusual when using rpath), ctypes is not able to dynamically load + # libraries installed with EasyBuild (see https://github.com/EESSI/software-layer/issues/192). + # If EasyBuild is configured to filter $LD_LIBRARY_PATH the patch specified in 'patch_ctypes_ld_library_path' + # are added to the list of patches. Also, we add the checksums_filter_ld_library_path to the checksums list in + # that case. + # This mechanism e.g. makes sure we can patch ctypes, which normally strongly relies on $LD_LIBRARY_PATH to find + # libraries. But, we want to do the patching conditionally on EasyBuild configuration (i.e. which env vars + # are filtered), hence this setup based on the custom easyconfig parameter 'patch_ctypes_ld_library_path' + filtered_env_vars = build_option('filter_env_vars') or [] + patch_ctypes_ld_library_path = self.cfg.get('patch_ctypes_ld_library_path') + if ( + 'LD_LIBRARY_PATH' in filtered_env_vars and + 'LIBRARY_PATH' not in filtered_env_vars and + patch_ctypes_ld_library_path + ): + # Some sanity checking so we can raise an early and clear error if needed + # We expect a (one) checksum for the patch_ctypes_ld_library_path + checksums = self.cfg['checksums'] + sources = self.cfg['sources'] + patches = self.cfg.get('patches') + len_patches = len(patches) if patches else 0 + if len_patches + len(sources) + 1 == len(checksums): + msg = "EasyBuild was configured to filter $LD_LIBRARY_PATH (and not to filter $LIBRARY_PATH). " + msg += "The ctypes module relies heavily on $LD_LIBRARY_PATH for locating its libraries. " + msg += "The following patch will be applied to make sure ctypes.CDLL, ctypes.cdll.LoadLibrary " + msg += f"and ctypes.util.find_library will still work correctly: {patch_ctypes_ld_library_path}." + self.log.info(msg) + self.log.info(f"Original list of patches: {self.cfg['patches']}") + self.log.info(f"Patch to be added: {patch_ctypes_ld_library_path}") + self.cfg.update('patches', [patch_ctypes_ld_library_path]) + self.log.info(f"Updated list of patches: {self.cfg['patches']}") + else: + msg = "The length of 'checksums' (%s) is not equal to the total amount of sources (%s) + patches (%s). " + msg += "Did you forget to add a checksum for patch_ctypes_ld_library_path?" + raise EasyBuildError(msg, len(checksums), len(sources), len(len_patches + 1)) + # If LD_LIBRARY_PATH is filtered, but no patch is specified, warn the user that his may not work + elif ( + 'LD_LIBRARY_PATH' in filtered_env_vars and + 'LIBRARY_PATH' not in filtered_env_vars and + not patch_ctypes_ld_library_path + ): + msg = "EasyBuild was configured to filter $LD_LIBRARY_PATH (and not to filter $LIBRARY_PATH). " + msg += "However, no patch for ctypes was specified through 'patch_ctypes_ld_library_path' in the " + msg += "easyconfig. Note that ctypes.util.find_library, ctypes.CDLL and ctypes.cdll.LoadLibrary heavily " + msg += "rely on $LD_LIBRARY_PATH. Without the patch, a setup without $LD_LIBRARY_PATH will likely not work " + msg += "correctly." + self.log.warning(msg) + + super().fetch_step(*args, **kwargs) + def patch_step(self, *args, **kwargs): """ Custom patch step for Python: @@ -357,33 +422,6 @@ def patch_step(self, *args, **kwargs): # Ignore user site dir. -E ignores PYTHONNOUSERSITE, so we have to add -s apply_regex_substitutions('configure', [(r"(PYTHON_FOR_BUILD=.*-E)'", r"\1 -s'")]) - # If we filter out LD_LIBRARY_PATH (not unusual when using rpath), ctypes is not able to dynamically load - # libraries installed with EasyBuild (see https://github.com/EESSI/software-layer/issues/192). - # ctypes is using GCC (and therefore LIBRARY_PATH) to figure out the full location but then only returns the - # soname, instead let's return the full path in this particular scenario - filtered_env_vars = build_option('filter_env_vars') or [] - if 'LD_LIBRARY_PATH' in filtered_env_vars and 'LIBRARY_PATH' not in filtered_env_vars: - ctypes_util_py = os.path.join("Lib", "ctypes", "util.py") - orig_gcc_so_name = None - # Let's do this incrementally since we are going back in time - if LooseVersion(self.version) >= "3.9.1": - # From 3.9.1 to at least v3.12.4 there is only one match for this line - orig_gcc_so_name = "_get_soname(_findLib_gcc(name)) or _get_soname(_findLib_ld(name))" - if orig_gcc_so_name: - orig_gcc_so_name_regex = r'(\s*)' + re.escape(orig_gcc_so_name) + r'(\s*)' - # _get_soname() takes the full path as an argument and uses objdump to get the SONAME field from - # the shared object file. The presence or absence of the SONAME field in the ELF header of a shared - # library is influenced by how the library is compiled and linked. For manually built libraries we - # may be lacking this field, this approach also solves that problem. - updated_gcc_so_name = ( - "_findLib_gcc(name) or _findLib_ld(name)" - ) - apply_regex_substitutions( - ctypes_util_py, - [(orig_gcc_so_name_regex, r'\1' + updated_gcc_so_name + r'\2')], - on_missing_match=ERROR - ) - # if we're installing Python with an alternate sysroot, # we need to patch setup.py which includes hardcoded paths like /usr/include and /lib64; # this fixes problems like not being able to build the _ssl module ("Could not build the ssl module") @@ -685,6 +723,57 @@ def install_step(self): symlink(target_lib_dynload, lib_dynload) change_dir(cwd) + def _sanity_check_ctypes_ld_library_path_patch(self): + """ + Check that ctypes.util.find_library and ctypes.CDLL work as expected. + When $LD_LIBRARY_PATH is filtered, a patch is required for this to work correctly + (see patch_ctypes_ld_library_path). + """ + # Try find_library first, since ctypes.CDLL relies on that to work correctly + cmd = "python -c 'from ctypes import util; print(util.find_library(\"libpython3.so\"))'" + res = run_shell_cmd(cmd) + out = res.output.strip() + escaped_python_root = re.escape(self.installdir) + pattern = rf"^{escaped_python_root}.*libpython3\.so$" + match = re.match(pattern, out) + self.log.debug(f"Matching regular expression pattern {pattern} to string {out}") + if match: + msg = "Call to ctypes.util.find_library('libpython3.so') successfully found libpython3.so under " + msg += f"the installation prefix of the current Python installation ({self.installdir}). " + if self.cfg.get('patch_ctypes_ld_library_path'): + msg += "This indicates that the patch that fixes ctypes when EasyBuild is " + msg += "configured to filter $LD_LIBRARY_PATH was applied succesfully." + self.log.info(msg) + else: + msg = "Finding the library libpython3.so using ctypes.util.find_library('libpython3.so') failed. " + msg += "The ctypes Python module requires a patch when EasyBuild is configured to filter $LD_LIBRARY_PATH. " + msg += "Please check if you specified a patch through patch_ctypes_ld_library_path and check " + msg += "the logs to see if it applied correctly." + raise EasyBuildError(msg) + # Now that we know find_library was patched correctly, check if ctypes.CDLL is also patched correctly + cmd = "python -c 'import ctypes; print(ctypes.CDLL(\"libpython3.so\"))'" + res = run_shell_cmd(cmd) + out = res.output.strip() + pattern = rf"^$" + match = re.match(pattern, out) + self.log.debug(f"Matching regular expression pattern {pattern} to string {out}") + if match: + msg = "Call to ctypes.CDLL('libpython3.so') succesfully opened libpython3.so. " + if self.cfg.get('patch_ctypes_ld_library_path'): + msg += "This indicates that the patch that fixes ctypes when $LD_LIBRARY_PATH is not set " + msg += "was applied successfully." + self.log.info(msg) + msg = "Call to ctypes.CDLL('libpython3.so') succesfully opened libpython3.so. " + if self.cfg.get('patch_ctypes_ld_library_path'): + msg += "This indicates that the patch that fixes ctypes when $LD_LIBRARY_PATH is not set " + msg += "was applied successfully." + else: + msg = "Opening of libpython3.so using ctypes.CDLL('libpython3.so') failed. " + msg += "The ctypes Python module requires a patch when EasyBuild is configured to filter $LD_LIBRARY_PATH. " + msg += "Please check if you specified a patch through patch_ctypes_ld_library_path and check " + msg += "the logs to see if it applied correctly." + raise EasyBuildError(msg) + def _sanity_check_ebpythonprefixes(self): """Check that EBPYTHONPREFIXES works""" temp_prefix = tempfile.mkdtemp(suffix='-tmp-prefix') @@ -749,6 +838,12 @@ def sanity_check_step(self): if self.cfg.get('ebpythonprefixes'): self._sanity_check_ebpythonprefixes() + # If the conditions for applying the patch specified through patch_ctypes_ld_library_path are met, + # check that a patch was applied and indeed fixed the issue + filtered_env_vars = build_option('filter_env_vars') or [] + if 'LD_LIBRARY_PATH' in filtered_env_vars and 'LIBRARY_PATH' not in filtered_env_vars: + self._sanity_check_ctypes_ld_library_path_patch() + pyver = 'python' + self.pyshortver custom_paths = { 'files': [