Delocate libraries into one directory per wheel.

Add a top-level package-less wheel to test.

Fixes duplicate library issues when a wheel has multiple packages.

Fixes issues where a wheel has no packages.

Changelog

@@ -25,6 +25,13 @@ Releases
 
   * ``wheel_libs`` now supports recursive dependencies.
   * ``listdeps`` command now supports recursive dependencies.
+  * Wheels with top-level modules can now be delocated.
+    [#72](https://github.com/matthew-brett/delocate/issues/72)
+    [#63](https://github.com/matthew-brett/delocate/issues/63)
+    [#45](https://github.com/matthew-brett/delocate/issues/45)
+    [#22](https://github.com/matthew-brett/delocate/issues/22)
+  * Wheels with multiple packages will no longer copy duplicate libraries.
+    [#35](https://github.com/matthew-brett/delocate/issues/35)
 
 * 0.9.1 (Friday September 17th 2021)
 

delocate/delocating.py

@@ -450,6 +450,37 @@ def _merge_lib_dict(d1, d2):
     return None
 
 
+def _decide_dylib_bundle_directory(
+    wheel_dir: str, package_name: str, lib_sdir: str = ".dylibs"
+) -> str:
+    """Return a relative directory which should be used to store dylib files.
+
+    Parameters
+    ----------
+    wheel_dir : str
+        The directory of an unpacked wheel to analyse.
+    package_name : str
+        The name of the package.
+    lib_sdir : str, optional
+        This value passed in via :func:`delocate_wheel`.
+
+    Returns
+    -------
+    dylibs_dir : str
+        A path to within `wheel_dir` where any library files should be put.
+    """
+    package_dirs = find_package_dirs(wheel_dir)
+    for directory in package_dirs:
+        if directory.endswith(package_name):
+            # Prefer using the directory with the same name as the package.
+            return pjoin(directory, lib_sdir)
+    if package_dirs:
+        # Otherwise, store dylib files in the first package alphabetically.
+        return pjoin(min(package_dirs), lib_sdir)
+    # Otherwise, use an auditwheel-style top-level name.
+    return pjoin(wheel_dir, f"{package_name}.dylibs")
+
+
 def delocate_wheel(
     in_wheel,  # type: Text
     out_wheel=None,  # type: Optional[Text]
@@ -479,6 +510,7 @@ def delocate_wheel(
     lib_sdir : str, optional
         Subdirectory name in wheel package directory (or directories) to store
         needed libraries.
+        This may be ignored depending on how the wheel is structured.
     lib_filt_func : None or str or callable, optional
         If None, inspect all files for dependencies on dynamic libraries. If
         callable, accepts filename as argument, returns True if we should
@@ -526,42 +558,44 @@ def delocate_wheel(
         all_copied = {}  # type: Dict[Text, Dict[Text, Text]]
         wheel_dir = realpath(pjoin(tmpdir, 'wheel'))
         zip2dir(in_wheel, wheel_dir)
-        for package_path in find_package_dirs(wheel_dir):
-            lib_path = pjoin(package_path, lib_sdir)
-            lib_path_exists = exists(lib_path)
-            copied_libs = delocate_path(
-                package_path,
-                lib_path,
-                lib_filt_func,
-                copy_filt_func,
-                executable_path=executable_path,
-                ignore_missing=ignore_missing,
-            )
-            if copied_libs and lib_path_exists:
+        # Assume the package name from the wheel filename.
+        package_name = basename(in_wheel).split("-")[0]
+        lib_sdir = _decide_dylib_bundle_directory(
+            wheel_dir, package_name, lib_sdir
+        )
+        lib_path = pjoin(wheel_dir, lib_sdir)
+        lib_path_exists = exists(lib_path)
+        copied_libs = delocate_path(
+            wheel_dir,
+            lib_path,
+            lib_filt_func,
+            copy_filt_func,
+            executable_path=executable_path,
+            ignore_missing=ignore_missing,
+        )
+        if copied_libs and lib_path_exists:
+            raise DelocationError(
+                '{0} already exists in wheel but need to copy '
+                '{1}'.format(lib_path, '; '.join(copied_libs)))
+        if len(os.listdir(lib_path)) == 0:
+            shutil.rmtree(lib_path)
+        # Check architectures
+        if require_archs is not None:
+            stop_fast = not check_verbose
+            bads = check_archs(copied_libs, require_archs, stop_fast)
+            if len(bads) != 0:
+                if check_verbose:
+                    print(bads_report(bads, pjoin(tmpdir, 'wheel')))
                 raise DelocationError(
-                    '{0} already exists in wheel but need to copy '
-                    '{1}'.format(lib_path, '; '.join(copied_libs)))
-            if len(os.listdir(lib_path)) == 0:
-                shutil.rmtree(lib_path)
-            # Check architectures
-            if require_archs is not None:
-                stop_fast = not check_verbose
-                bads = check_archs(copied_libs, require_archs, stop_fast)
-                if len(bads) != 0:
-                    if check_verbose:
-                        print(bads_report(bads, pjoin(tmpdir, 'wheel')))
-                    raise DelocationError(
-                        "Some missing architectures in wheel")
-            # Change install ids to be unique within Python space
-            install_id_root = (DLC_PREFIX +
-                               relpath(package_path, wheel_dir) +
-                               '/')
-            for lib in copied_libs:
-                lib_base = basename(lib)
-                copied_path = pjoin(lib_path, lib_base)
-                set_install_id(copied_path, install_id_root + lib_base)
-                validate_signature(copied_path)
-            _merge_lib_dict(all_copied, copied_libs)
+                    "Some missing architectures in wheel")
+        # Change install ids to be unique within Python space
+        install_id_root = DLC_PREFIX + relpath(lib_sdir, wheel_dir) + '/'
+        for lib in copied_libs:
+            lib_base = basename(lib)
+            copied_path = pjoin(lib_path, lib_base)
+            set_install_id(copied_path, install_id_root + lib_base)
+            validate_signature(copied_path)
+        _merge_lib_dict(all_copied, copied_libs)
         if len(all_copied):
             rewrite_record(wheel_dir)
         if len(all_copied) or not in_place:

delocate/tests/test_wheelies.py

@@ -38,6 +38,7 @@ def _collect_wheel(globber):
 PLAT_WHEEL = _collect_wheel('fakepkg1-1.0-cp*.whl')
 PURE_WHEEL = _collect_wheel('fakepkg2-1.0-py*.whl')
 RPATH_WHEEL = _collect_wheel('fakepkg_rpath-1.0-cp*.whl')
+TOPLEVEL_WHEEL = _collect_wheel('fakepkg_toplevel-1.0-cp*.whl')
 STRAY_LIB = pjoin(DATA_PATH, 'libextfunc.dylib')
 # The install_name in the wheel for the stray library
 STRAY_LIB_DEP = realpath(STRAY_LIB)
@@ -140,8 +141,8 @@ def test_fix_plat():
         zip2dir(fixed_wheel, 'plat_pkg3')
         base_stray = basename(stray_lib)
         the_lib = pjoin('plat_pkg3', 'fakepkg1', '.dylibs', base_stray)
-        inst_id = DLC_PREFIX + 'fakepkg1/' + base_stray
-        assert_equal(get_install_id(the_lib), inst_id)
+        inst_id = DLC_PREFIX + 'fakepkg1/.dylibs/' + base_stray
+        assert get_install_id(the_lib) == inst_id
 
 
 def test_script_permissions():
@@ -342,3 +343,22 @@ def ignore_libextfunc2(path: str) -> bool:
         assert delocate_wheel(
             RPATH_WHEEL, 'tmp.whl', lib_filt_func=ignore_libextfunc2
         ) == stray_libs_only_direct
+
+
+def test_fix_toplevel() -> None:
+    # Test wheels which are not organized into packages.
+
+    with InTemporaryDirectory():
+        # The module was set to expect its dependency in the libs/ directory
+        os.symlink(DATA_PATH, 'libs')
+
+        dep_mod = 'module2.abi3.so'
+        dep_path = '@rpath/libextfunc2_rpath.dylib'
+
+        stray_libs = {
+            realpath('libs/libextfunc2_rpath.dylib'): {dep_mod: dep_path},
+        }
+
+        assert delocate_wheel(TOPLEVEL_WHEEL, 'out.whl') == stray_libs
+        with InWheel("out.whl") as wheel_path:
+            assert "fakepkg_toplevel.dylibs" in os.listdir(wheel_path)

delocate/tools.py

@@ -4,6 +4,7 @@
 
 import os
 from os.path import join as pjoin, relpath, isdir, exists
+from typing import Set
 import zipfile
 import re
 import stat
@@ -443,7 +444,7 @@ def dir2zip(in_dir, zip_fname):
     z.close()
 
 
-def find_package_dirs(root_path):
+def find_package_dirs(root_path: str) -> Set[str]:
     """ Find python package directories in directory `root_path`
 
     Parameters

wheel_makers/fakepkg_toplevel/README.md

@@ -0,0 +1,3 @@
+This contains a single compiled module not stored inside of a Python package.
+
+This reuses the `libextfunc2.dylib` library from the `fakepkg_rpath` wheel.

wheel_makers/fakepkg_toplevel/libs/extfunc2.c

@@ -0,0 +1,4 @@
+int extfunc2()
+{
+    return 3;
+}

wheel_makers/fakepkg_toplevel/module2.c

@@ -0,0 +1,32 @@
+#define PY_SSIZE_T_CLEAN
+#include <Python.h>
+
+int extfunc2(void); /*proto*/
+
+static PyObject* func2(PyObject* self) {
+    return PyLong_FromLong(2);
+}
+
+static PyObject* func3(PyObject* self) {
+    return PyLong_FromLong((long)extfunc2());
+}
+
+static PyMethodDef module2_methods[] = {
+    {"func2", (PyCFunction)func2, METH_NOARGS, NULL},
+    {"func3", (PyCFunction)func3, METH_NOARGS, NULL},
+    {NULL, NULL, 0, NULL}   /* sentinel */
+};
+
+static struct PyModuleDef module2 = {
+    PyModuleDef_HEAD_INIT,
+    "module2",
+    NULL,
+    -1,
+    module2_methods
+};
+
+PyMODINIT_FUNC
+PyInit_module2(void)
+{
+    return PyModule_Create(&module2);
+}

wheel_makers/fakepkg_toplevel/setup.py

@@ -0,0 +1,40 @@
+import subprocess
+from pathlib import Path
+
+from setuptools import Extension, setup  # type: ignore
+
+HERE = Path(__file__).parent.resolve(strict=True)
+LIBS = HERE / "libs"
+ARCH_FLAGS = ["-arch", "arm64", "-arch", "x86_64"]  # Dual architecture.
+
+subprocess.run(
+    [
+        "cc",
+        str(LIBS / "extfunc2.c"),
+        "-dynamiclib",
+        "-install_name",
+        "@rpath/libextfunc2_rpath.dylib",
+        "-o",
+        str(LIBS / "libextfunc2_rpath.dylib"),
+    ]
+    + ARCH_FLAGS,
+    check=True,
+)
+
+ext_modules = [
+    Extension(
+        "module2",
+        ["module2.c"],
+        libraries=["extfunc2_rpath"],
+        extra_compile_args=ARCH_FLAGS,
+        extra_link_args=[f"-L{LIBS}", "-rpath", "libs/"],
+        py_limited_api=True,
+    )
+]
+
+setup(
+    ext_modules=ext_modules,
+    name="fakepkg_toplevel",
+    version="1.0",
+    py_modules=["module2", "test_fakepkg_toplevel"],
+)

wheel_makers/fakepkg_toplevel/test_fakepkg_toplevel.py

@@ -0,0 +1,6 @@
+from module2 import func2, func3  # type: ignore
+
+
+def test_fakepkg():
+    assert func2() == 2
+    assert func3() == 3

wheel_makers/make_wheels.sh

@@ -24,6 +24,10 @@ cd fakepkg_rpath
 python setup.py clean bdist_wheel --py-limited-api=cp36
 cd -
 
+cd fakepkg_toplevel
+python setup.py clean bdist_wheel --py-limited-api=cp36
+cd -
+
 OUT_PATH=../delocate/tests/data
 rm $OUT_PATH/fakepkg*.whl
 cp */dist/*.whl $OUT_PATH