Fix loading system libraries with ctypes on macOS Big Sur.

On Big Sur, there is no longer a separate dylib file for each system
library. Instead, they have been combined into a single shared cache
file. As a result, it no longer suffices to check file existence to
determine whether a system library is present. Fix this by calling
_dyld_shared_cache_contains_path to check the dyld shared cache as well.

The changes under lib-python/ are copied from
python/cpython#22855.

The _dyld_shared_cache_contains_path function is exposed to the ctypes
python code using CFFI, similar to what is done in other modules, such
as lib_pypy/_sha3.

Fixes issue #3314.

Author: rickyz

lib-python/3/ctypes/macholib/dyld.py

@@ -6,6 +6,11 @@
 from ctypes.macholib.framework import framework_info
 from ctypes.macholib.dylib import dylib_info
 from itertools import *
+try:
+    from _ctypes import _dyld_shared_cache_contains_path
+except ImportError:
+    def _dyld_shared_cache_contains_path(*args):
+        raise NotImplementedError
 
 __all__ = [
     'dyld_find', 'framework_find',
@@ -122,8 +127,15 @@ def dyld_find(name, executable_path=None, env=None):
                 dyld_executable_path_search(name, executable_path),
                 dyld_default_search(name, env),
             ), env):
+
         if os.path.isfile(path):
             return path
+        try:
+            if _dyld_shared_cache_contains_path(path):
+                return path
+        except NotImplementedError:
+            pass
+
     raise ValueError("dylib %s could not be found" % (name,))
 
 def framework_find(fn, executable_path=None, env=None):

lib-python/3/ctypes/test/test_macholib.py

@@ -45,19 +45,22 @@ def find_lib(name):
 class MachOTest(unittest.TestCase):
     @unittest.skipUnless(sys.platform == "darwin", 'OSX-specific test')
     def test_find(self):
-
-        self.assertEqual(find_lib('pthread'),
-                             '/usr/lib/libSystem.B.dylib')
+        # On Mac OS 11, system dylibs are only present in the shared cache,
+        # so symlinks like libpthread.dylib -> libSystem.B.dylib will not
+        # be resolved by dyld_find
+        self.assertIn(find_lib('pthread'),
+                              ('/usr/lib/libSystem.B.dylib', '/usr/lib/libpthread.dylib'))
 
         result = find_lib('z')
         # Issue #21093: dyld default search path includes $HOME/lib and
         # /usr/local/lib before /usr/lib, which caused test failures if
         # a local copy of libz exists in one of them. Now ignore the head
         # of the path.
-        self.assertRegex(result, r".*/lib/libz\..*.*\.dylib")
+        self.assertRegex(result, r".*/lib/libz.*\.dylib")
 
-        self.assertEqual(find_lib('IOKit'),
-                             '/System/Library/Frameworks/IOKit.framework/Versions/A/IOKit')
+        self.assertIn(find_lib('IOKit'),
+                              ('/System/Library/Frameworks/IOKit.framework/Versions/A/IOKit',
+                              '/System/Library/Frameworks/IOKit.framework/IOKit'))
 
 if __name__ == "__main__":
     unittest.main()

lib_pypy/_ctypes/__init__.py

@@ -13,15 +13,15 @@
     _string_at_addr, _wstring_at_addr, set_conversion_mode)
 from _ctypes.union import Union
 
+try: from __pypy__ import builtinify
+except ImportError: builtinify = lambda f: f
+
 import os as _os
 
 if _os.name in ("nt", "ce"):
     from _rawffi import FormatError
     from _rawffi import check_HRESULT as _check_HRESULT
 
-    try: from __pypy__ import builtinify
-    except ImportError: builtinify = lambda f: f
-
     @builtinify
     def CopyComPointer(src, dst):
         from ctypes import c_void_p, cast
@@ -32,8 +32,6 @@ def CopyComPointer(src, dst):
         dst[0] = cast(src, c_void_p).value
         return 0
 
-    del builtinify
-
     LoadLibrary = dlopen
 
 from _rawffi import FUNCFLAG_STDCALL, FUNCFLAG_CDECL, FUNCFLAG_PYTHONAPI
@@ -43,6 +41,20 @@ def CopyComPointer(src, dst):
 if _os.name in ("nt", "ce"):
     from _ctypes.builtin import get_last_error, set_last_error
 
+import sys as _sys
+if _sys.platform == 'darwin':
+    try:
+        from ._ctypes_cffi import ffi as _ffi, lib as _lib
+        @builtinify
+        def _dyld_shared_cache_contains_path(path):
+            if not hasattr(_lib, '_dyld_shared_cache_contains_path'):
+                raise NotImplementedError
+            return _lib._dyld_shared_cache_contains_path(path.encode())
+    except ImportError:
+        pass
+
+del builtinify
+
 __version__ = '1.1.0'
 #XXX platform dependant?
 RTLD_LOCAL = 0

lib_pypy/_ctypes/_ctypes_build.py

@@ -0,0 +1,23 @@
+import os
+import platform
+
+from cffi import FFI
+ffi = FFI()
+
+def main():
+    if platform.system() != 'Darwin':
+        return
+
+    release, _, _ = platform.mac_ver()
+    release = tuple(map(int, release.split('.')))
+    if release < (10, 16):
+        return
+
+    ffi.cdef('bool _dyld_shared_cache_contains_path(const char* path);')
+    ffi.set_source('_ctypes_cffi', '#include <mach-o/dyld.h>')
+
+    os.chdir(os.path.dirname(__file__))
+    ffi.compile()
+
+if __name__ == '__main__':
+    main()

lib_pypy/pypy_tools/build_cffi_imports.py

@@ -23,6 +23,7 @@ class MissingDependenciesError(Exception):
 
 
 cffi_build_scripts = {
+    "_ctypes": "_ctypes/_ctypes_build.py" if sys.platform == "darwin" else None,
     "_blake2": "_blake2/_blake2_build.py",
     "_ssl": "_ssl_build.py",
     "sqlite3": "_sqlite3_build.py",