add support to Python easyblock for conditionally applying extra patch to fix ctypes when EasyBuild is configured to filter $LD_LIBRARY_PATH

easybuild/easyblocks/p/python.py

@@ -302,6 +302,13 @@ def extra_options():
             'ulimit_unlimited': [False, "Ensure stack size limit is set to '%s' during build" % UNLIMITED, CUSTOM],
             'use_lto': [None, "Build with Link Time Optimization (>= v3.7.0, potentially unstable on some toolchains). "
                         "If None: auto-detect based on toolchain compiler (version)", CUSTOM],
+            'patch_ctypes_ld_library_path': [None,
+                                             "The ctypes module strongly relies on LD_LIBRARY_PATH to find "
+                                             "libraries. This allows specifying a patch that will only be "
+                                             "applied if EasyBuild is configured to filter LD_LIBRARY_PATH, in "
+                                             "order to make sure ctypes can still find libraries without it. "
+                                             "Please make sure to add the checksum for this patch to 'checksums'.",
+                                             CUSTOM],
         }
         return ConfigureMake.extra_options(extra_vars)
 
@@ -345,6 +352,64 @@ def _get_pip_ext_version(self):
                 return ext[1]
         return None
 
+    def fetch_step(self, *args, **kwargs):
+        """
+        Custom fetch step for Python.
+
+        Add patch specified in patch_ctypes_ld_library_path to list of patches if
+        EasyBuild is configured to filter $LD_LIBRARY_PATH (and is configured not to filter $LIBRARY_PATH).
+        This needs to be done in (or before) the fetch step to ensure that those patches are also fetched.
+        """
+        # If we filter out $LD_LIBRARY_PATH (not unusual when using rpath), ctypes is not able to dynamically load
+        # libraries installed with EasyBuild (see https://github.com/EESSI/software-layer/issues/192).
+        # If EasyBuild is configured to filter $LD_LIBRARY_PATH the patch specified in 'patch_ctypes_ld_library_path'
+        # are added to the list of patches. Also, we add the checksums_filter_ld_library_path to the checksums list in
+        # that case.
+        # This mechanism e.g. makes sure we can patch ctypes, which normally strongly relies on $LD_LIBRARY_PATH to find
+        # libraries. But, we want to do the patching conditionally on EasyBuild configuration (i.e. which env vars
+        # are filtered), hence this setup based on the custom easyconfig parameter 'patch_ctypes_ld_library_path'
+        filtered_env_vars = build_option('filter_env_vars') or []
+        patch_ctypes_ld_library_path = self.cfg.get('patch_ctypes_ld_library_path')
+        if (
+            'LD_LIBRARY_PATH' in filtered_env_vars and
+            'LIBRARY_PATH' not in filtered_env_vars and
+            patch_ctypes_ld_library_path
+        ):
+            # Some sanity checking so we can raise an early and clear error if needed
+            # We expect a (one) checksum for the patch_ctypes_ld_library_path
+            checksums = self.cfg['checksums']
+            sources = self.cfg['sources']
+            patches = self.cfg.get('patches')
+            len_patches = len(patches) if patches else 0
+            if len_patches + len(sources) + 1 == len(checksums):
+                msg = "EasyBuild was configured to filter $LD_LIBRARY_PATH (and not to filter $LIBRARY_PATH). "
+                msg += "The ctypes module relies heavily on $LD_LIBRARY_PATH for locating its libraries. "
+                msg += "The following patch will be applied to make sure ctypes.CDLL, ctypes.cdll.LoadLibrary "
+                msg += f"and ctypes.util.find_library will still work correctly: {patch_ctypes_ld_library_path}."
+                self.log.info(msg)
+                self.log.info(f"Original list of patches: {self.cfg['patches']}")
+                self.log.info(f"Patch to be added: {patch_ctypes_ld_library_path}")
+                self.cfg.update('patches', [patch_ctypes_ld_library_path])
+                self.log.info(f"Updated list of patches: {self.cfg['patches']}")
+            else:
+                msg = "The length of 'checksums' (%s) is not equal to the total amount of sources (%s) + patches (%s). "
+                msg += "Did you forget to add a checksum for patch_ctypes_ld_library_path?"
+                raise EasyBuildError(msg, len(checksums), len(sources), len(len_patches + 1))
+        # If LD_LIBRARY_PATH is filtered, but no patch is specified, warn the user that his may not work
+        elif (
+            'LD_LIBRARY_PATH' in filtered_env_vars and
+            'LIBRARY_PATH' not in filtered_env_vars and
+            not patch_ctypes_ld_library_path
+        ):
+            msg = "EasyBuild was configured to filter $LD_LIBRARY_PATH (and not to filter $LIBRARY_PATH). "
+            msg += "However, no patch for ctypes was specified through 'patch_ctypes_ld_library_path' in the "
+            msg += "easyconfig. Note that ctypes.util.find_library, ctypes.CDLL and ctypes.cdll.LoadLibrary heavily "
+            msg += "rely on $LD_LIBRARY_PATH. Without the patch, a setup without $LD_LIBRARY_PATH will likely not work "
+            msg += "correctly."
+            self.log.warning(msg)
+
+        super().fetch_step(*args, **kwargs)
+
     def patch_step(self, *args, **kwargs):
         """
         Custom patch step for Python:
@@ -357,33 +422,6 @@ def patch_step(self, *args, **kwargs):
             # Ignore user site dir. -E ignores PYTHONNOUSERSITE, so we have to add -s
             apply_regex_substitutions('configure', [(r"(PYTHON_FOR_BUILD=.*-E)'", r"\1 -s'")])
 
-        # If we filter out LD_LIBRARY_PATH (not unusual when using rpath), ctypes is not able to dynamically load
-        # libraries installed with EasyBuild (see https://github.com/EESSI/software-layer/issues/192).
-        # ctypes is using GCC (and therefore LIBRARY_PATH) to figure out the full location but then only returns the
-        # soname, instead let's return the full path in this particular scenario
-        filtered_env_vars = build_option('filter_env_vars') or []
-        if 'LD_LIBRARY_PATH' in filtered_env_vars and 'LIBRARY_PATH' not in filtered_env_vars:
-            ctypes_util_py = os.path.join("Lib", "ctypes", "util.py")
-            orig_gcc_so_name = None
-            # Let's do this incrementally since we are going back in time
-            if LooseVersion(self.version) >= "3.9.1":
-                # From 3.9.1 to at least v3.12.4 there is only one match for this line
-                orig_gcc_so_name = "_get_soname(_findLib_gcc(name)) or _get_soname(_findLib_ld(name))"
-            if orig_gcc_so_name:
-                orig_gcc_so_name_regex = r'(\s*)' + re.escape(orig_gcc_so_name) + r'(\s*)'
-                # _get_soname() takes the full path as an argument and uses objdump to get the SONAME field from
-                # the shared object file. The presence or absence of the SONAME field in the ELF header of a shared
-                # library is influenced by how the library is compiled and linked. For manually built libraries we
-                # may be lacking this field, this approach also solves that problem.
-                updated_gcc_so_name = (
-                    "_findLib_gcc(name) or _findLib_ld(name)"
-                )
-                apply_regex_substitutions(
-                    ctypes_util_py,
-                    [(orig_gcc_so_name_regex, r'\1' + updated_gcc_so_name + r'\2')],
-                    on_missing_match=ERROR
-                )
-
         # if we're installing Python with an alternate sysroot,
         # we need to patch setup.py which includes hardcoded paths like /usr/include and /lib64;
         # this fixes problems like not being able to build the _ssl module ("Could not build the ssl module")
@@ -685,6 +723,57 @@ def install_step(self):
                 symlink(target_lib_dynload, lib_dynload)
                 change_dir(cwd)
 
+    def _sanity_check_ctypes_ld_library_path_patch(self):
+        """
+        Check that ctypes.util.find_library and ctypes.CDLL work as expected.
+        When $LD_LIBRARY_PATH is filtered, a patch is required for this to work correctly
+        (see patch_ctypes_ld_library_path).
+        """
+        # Try find_library first, since ctypes.CDLL relies on that to work correctly
+        cmd = "python -c 'from ctypes import util; print(util.find_library(\"libpython3.so\"))'"
+        res = run_shell_cmd(cmd)
+        out = res.output.strip()
+        escaped_python_root = re.escape(self.installdir)
+        pattern = rf"^{escaped_python_root}.*libpython3\.so$"
+        match = re.match(pattern, out)
+        self.log.debug(f"Matching regular expression pattern {pattern} to string {out}")
+        if match:
+            msg = "Call to ctypes.util.find_library('libpython3.so') successfully found libpython3.so under "
+            msg += f"the installation prefix of the current Python installation ({self.installdir}). "
+            if self.cfg.get('patch_ctypes_ld_library_path'):
+                msg += "This indicates that the patch that fixes ctypes when EasyBuild is "
+                msg += "configured to filter $LD_LIBRARY_PATH was applied succesfully."
+            self.log.info(msg)
+        else:
+            msg = "Finding the library libpython3.so using ctypes.util.find_library('libpython3.so') failed. "
+            msg += "The ctypes Python module requires a patch when EasyBuild is configured to filter $LD_LIBRARY_PATH. "
+            msg += "Please check if you specified a patch through patch_ctypes_ld_library_path and check "
+            msg += "the logs to see if it applied correctly."
+            raise EasyBuildError(msg)
+        # Now that we know find_library was patched correctly, check if ctypes.CDLL is also patched correctly
+        cmd = "python -c 'import ctypes; print(ctypes.CDLL(\"libpython3.so\"))'"
+        res = run_shell_cmd(cmd)
+        out = res.output.strip()
+        pattern = rf"^<CDLL '{escaped_python_root}.*libpython3\.so', handle [a-f0-9]+ at 0x[a-f0-9]+>$"
+        match = re.match(pattern, out)
+        self.log.debug(f"Matching regular expression pattern {pattern} to string {out}")
+        if match:
+            msg = "Call to ctypes.CDLL('libpython3.so') succesfully opened libpython3.so. "
+            if self.cfg.get('patch_ctypes_ld_library_path'):
+                msg += "This indicates that the patch that fixes ctypes when $LD_LIBRARY_PATH is not set "
+                msg += "was applied successfully."
+            self.log.info(msg)
+            msg = "Call to ctypes.CDLL('libpython3.so') succesfully opened libpython3.so. "
+            if self.cfg.get('patch_ctypes_ld_library_path'):
+                msg += "This indicates that the patch that fixes ctypes when $LD_LIBRARY_PATH is not set "
+                msg += "was applied successfully."
+        else:
+            msg = "Opening of libpython3.so using ctypes.CDLL('libpython3.so') failed. "
+            msg += "The ctypes Python module requires a patch when EasyBuild is configured to filter $LD_LIBRARY_PATH. "
+            msg += "Please check if you specified a patch through patch_ctypes_ld_library_path and check "
+            msg += "the logs to see if it applied correctly."
+            raise EasyBuildError(msg)
+
     def _sanity_check_ebpythonprefixes(self):
         """Check that EBPYTHONPREFIXES works"""
         temp_prefix = tempfile.mkdtemp(suffix='-tmp-prefix')
@@ -749,6 +838,12 @@ def sanity_check_step(self):
         if self.cfg.get('ebpythonprefixes'):
             self._sanity_check_ebpythonprefixes()
 
+        # If the conditions for applying the patch specified through patch_ctypes_ld_library_path are met,
+        # check that a patch was applied and indeed fixed the issue
+        filtered_env_vars = build_option('filter_env_vars') or []
+        if 'LD_LIBRARY_PATH' in filtered_env_vars and 'LIBRARY_PATH' not in filtered_env_vars:
+            self._sanity_check_ctypes_ld_library_path_patch()
+
         pyver = 'python' + self.pyshortver
         custom_paths = {
             'files': [