From a0a78b42bc835a408961bcb44765520f26a13ad3 Mon Sep 17 00:00:00 2001 From: Malcolm Smith Date: Tue, 16 Dec 2025 11:56:10 +0000 Subject: [PATCH 01/14] Use auditwheel on Android --- bin/generate_schema.py | 17 +- cibuildwheel/platforms/android.py | 152 +++--------------- .../resources/cibuildwheel.schema.json | 15 +- .../resources/constraints-python310.txt | 2 + .../resources/constraints-python311.txt | 2 + .../resources/constraints-python312.txt | 2 + .../resources/constraints-python313.txt | 2 + .../resources/constraints-python314.txt | 2 + .../resources/constraints-python38.txt | 2 + .../resources/constraints-python39.txt | 2 + cibuildwheel/resources/constraints.in | 1 + cibuildwheel/resources/constraints.txt | 2 + docs/options.md | 4 +- pyproject.toml | 2 - 14 files changed, 68 insertions(+), 139 deletions(-) diff --git a/bin/generate_schema.py b/bin/generate_schema.py index 611d19e5a..f6f6072bd 100755 --- a/bin/generate_schema.py +++ b/bin/generate_schema.py @@ -366,14 +366,15 @@ def as_object(d: dict[str, Any]) -> dict[str, Any]: "ios": as_object(not_linux), } -oses["linux"]["properties"]["repair-wheel-command"] = { - **schema["properties"]["repair-wheel-command"], - "default": "auditwheel repair -w {dest_dir} {wheel}", -} -oses["macos"]["properties"]["repair-wheel-command"] = { - **schema["properties"]["repair-wheel-command"], - "default": "delocate-wheel --require-archs {delocate_archs} -w {dest_dir} -v {wheel}", -} +for os_name, command in [ + ("linux", "auditwheel repair -w {dest_dir} {wheel}"), + ("macos", "delocate-wheel --require-archs {delocate_archs} -w {dest_dir} -v {wheel}"), + ("android", "auditwheel repair --ldpaths {ldpaths} -w {dest_dir} {wheel}"), +]: + oses[os_name]["properties"]["repair-wheel-command"] = { + **schema["properties"]["repair-wheel-command"], + "default": command, + } del oses["linux"]["properties"]["dependency-versions"] diff --git a/cibuildwheel/platforms/android.py b/cibuildwheel/platforms/android.py index 229401e4f..63eb3d770 100644 --- a/cibuildwheel/platforms/android.py +++ b/cibuildwheel/platforms/android.py @@ -1,15 +1,10 @@ -import csv -import hashlib import os import platform import re import shlex import shutil import subprocess -import sysconfig -from collections.abc import Iterable, Iterator from dataclasses import dataclass -from os.path import relpath from pathlib import Path from pprint import pprint from runpy import run_path @@ -18,8 +13,6 @@ from build import ProjectBuilder from build.env import IsolatedEnv -from elftools.common.exceptions import ELFError -from elftools.elf.elffile import ELFFile from filelock import FileLock from .. import errors, platforms # pylint: disable=cyclic-import @@ -228,10 +221,19 @@ def setup_env( android_env = setup_android_env(config, python_dir, venv_dir, build_env) # Install build tools - if build_frontend not in {"build", "build[uv]"}: + # TODO: use an official auditwheel version once + # https://github.com/pypa/auditwheel/pull/643 has been released, and add it to the + # constraints files. + tools = [ + "auditwheel @ git+https://github.com/mhsmith/auditwheel@android", + "patchelf", + ] + if build_frontend in {"build", "build[uv]"}: + tools.append("build") + else: msg = "Android requires the build frontend to be 'build'" raise errors.FatalError(msg) - call(*pip, "install", "build", *constraint_flags(dependency_constraint), env=build_env) + call(*pip, "install", *tools, *constraint_flags(dependency_constraint), env=build_env) # Build-time requirements must be queried within android_env, because # `get_requires_for_build` can run arbitrary code in setup.py scripts, which may be @@ -321,6 +323,8 @@ def localized_vars( if isinstance(final, str): final = final.replace(orig_prefix, str(prefix)) + # By default we build against the same API level as Python itself, but this can + # be overridden with an environment variable. if key == "ANDROID_API_LEVEL": if api_level := build_env.get(key): final = int(api_level) @@ -445,9 +449,20 @@ def repair_wheel(state: BuildState, built_wheel: Path) -> Path: repaired_wheel_dir.mkdir() if state.options.repair_command: + toolchain = Path(state.android_env["CC"]).parent.parent + ldpaths = ":".join( + # In the future, we may use this to implement PEP 725 by installing + # libraries in {state.python_dir}/prefix/lib or elsewhere, and adding that + # location to ldpaths. + [ + # For libc++_shared. + f"{toolchain}/sysroot/usr/lib/{state.android_env['CIBW_HOST_TRIPLET']}", + ] + ) shell( prepare_command( state.options.repair_command, + ldpaths=ldpaths, wheel=built_wheel, dest_dir=repaired_wheel_dir, package=state.options.package_dir, @@ -456,7 +471,7 @@ def repair_wheel(state: BuildState, built_wheel: Path) -> Path: env=state.build_env, ) else: - repair_default(state.android_env, built_wheel, repaired_wheel_dir) + shutil.move(built_wheel, repaired_wheel_dir) repaired_wheels = list(repaired_wheel_dir.glob("*.whl")) if len(repaired_wheels) == 0: @@ -471,106 +486,6 @@ def repair_wheel(state: BuildState, built_wheel: Path) -> Path: return repaired_wheel -def repair_default( - android_env: dict[str, str], built_wheel: Path, repaired_wheel_dir: Path -) -> None: - """ - Adds libc++ to the wheel if anything links against it. In the future this should be - moved to auditwheel and generalized to support more libraries. - """ - if (match := re.search(r"^(.+?)-", built_wheel.name)) is None: - msg = f"Failed to parse wheel filename: {built_wheel.name}" - raise errors.FatalError(msg) - wheel_name = match[1] - - unpacked_dir = repaired_wheel_dir / "unpacked" - unpacked_dir.mkdir() - shutil.unpack_archive(built_wheel, unpacked_dir, format="zip") - - # Some build systems are inconsistent about name normalization, so don't assume the - # dist-info name is identical to the wheel name. - record_paths = list(unpacked_dir.glob("*.dist-info/RECORD")) - if len(record_paths) != 1: - msg = f"{built_wheel.name} contains {len(record_paths)} dist-info/RECORD files; expected 1" - raise errors.FatalError(msg) - - old_soname = "libc++_shared.so" - paths_to_patch = [] - for path, elffile in elf_file_filter( - unpacked_dir / filename - for filename, *_ in csv.reader(record_paths[0].read_text().splitlines()) - ): - if (dynamic := elffile.get_section_by_name(".dynamic")) and any( # type: ignore[no-untyped-call] - tag.entry.d_tag == "DT_NEEDED" and tag.needed == old_soname - for tag in dynamic.iter_tags() - ): - paths_to_patch.append(path) - - if not paths_to_patch: - shutil.copyfile(built_wheel, repaired_wheel_dir / built_wheel.name) - else: - # Android doesn't support DT_RPATH, but supports DT_RUNPATH since API level 24 - # (https://github.com/aosp-mirror/platform_bionic/blob/master/android-changes-for-ndk-developers.md). - if int(sysconfig_print('get_config_vars()["ANDROID_API_LEVEL"]', android_env)) < 24: - msg = f"Adding {old_soname} requires ANDROID_API_LEVEL to be at least 24" - raise errors.FatalError(msg) - - toolchain = Path(android_env["CC"]).parent.parent - src_path = toolchain / f"sysroot/usr/lib/{android_env['CIBW_HOST_TRIPLET']}/{old_soname}" - - # Use the same library location as auditwheel would. - libs_dir = unpacked_dir / (wheel_name + ".libs") - libs_dir.mkdir() - new_soname = soname_with_hash(src_path) - dst_path = libs_dir / new_soname - shutil.copyfile(src_path, dst_path) - call(which("patchelf"), "--set-soname", new_soname, dst_path) - - for path in paths_to_patch: - call(which("patchelf"), "--replace-needed", old_soname, new_soname, path) - call( - which("patchelf"), - "--set-rpath", - f"${{ORIGIN}}/{relpath(libs_dir, path.parent)}", - path, - ) - call(which("wheel"), "pack", unpacked_dir, "-d", repaired_wheel_dir) - - -# If cibuildwheel was called without activating its environment, its scripts directory -# will not be on the PATH. -def which(cmd: str) -> str: - scripts_dir = sysconfig.get_path("scripts") - result = shutil.which(cmd, path=scripts_dir + os.pathsep + os.environ["PATH"]) - if result is None: - msg = f"Couldn't find {cmd!r} in {scripts_dir} or on the PATH" - raise errors.FatalError(msg) - return result - - -def elf_file_filter(paths: Iterable[Path]) -> Iterator[tuple[Path, ELFFile]]: - """Filter through an iterator of filenames and load up only ELF files""" - for path in paths: - if not path.name.endswith(".py"): - try: - with open(path, "rb") as f: - candidate = ELFFile(f) # type: ignore[no-untyped-call] - yield path, candidate - except ELFError: - pass # Not an ELF file - - -def soname_with_hash(src_path: Path) -> str: - """Return the same library filename as auditwheel would""" - shorthash = hashlib.sha256(src_path.read_bytes()).hexdigest()[:8] - src_name = src_path.name - base, ext = src_name.split(".", 1) - if not base.endswith(f"-{shorthash}"): - return f"{base}-{shorthash}.{ext}" - else: - return src_name - - def test_wheel(state: BuildState, wheel: Path, *, build_frontend: str) -> None: test_command = state.options.test_command if not (test_command and state.options.test_selector(state.config.identifier)): @@ -600,12 +515,7 @@ def test_wheel(state: BuildState, wheel: Path, *, build_frontend: str) -> None: ) platform_args = ( - ["--python-platform", android_triplet(state.config.identifier)] - if use_uv - else [ - "--platform", - sysconfig_print("get_platform()", state.android_env).replace("-", "_"), - ] + ["--python-platform", android_triplet(state.config.identifier)] if use_uv else [] ) # Install the wheel and test-requires. @@ -690,13 +600,3 @@ def test_wheel(state: BuildState, wheel: Path, *, build_frontend: str) -> None: *test_args, env=state.build_env, ) - - -def sysconfig_print(method_call: str, env: dict[str, str]) -> str: - return call( - "python", - "-c", - f'import sysconfig; print(sysconfig.{method_call}, end="")', - env=env, - capture_stdout=True, - ) diff --git a/cibuildwheel/resources/cibuildwheel.schema.json b/cibuildwheel/resources/cibuildwheel.schema.json index 2c52aada4..a1b1ca6a4 100644 --- a/cibuildwheel/resources/cibuildwheel.schema.json +++ b/cibuildwheel/resources/cibuildwheel.schema.json @@ -1161,7 +1161,20 @@ "$ref": "#/properties/pyodide-version" }, "repair-wheel-command": { - "$ref": "#/properties/repair-wheel-command" + "description": "Execute a shell command to repair each built wheel.", + "oneOf": [ + { + "type": "string" + }, + { + "type": "array", + "items": { + "type": "string" + } + } + ], + "title": "CIBW_REPAIR_WHEEL_COMMAND", + "default": "auditwheel repair --ldpaths {ldpaths} -w {dest_dir} {wheel}" }, "test-command": { "$ref": "#/properties/test-command" diff --git a/cibuildwheel/resources/constraints-python310.txt b/cibuildwheel/resources/constraints-python310.txt index 698481a26..94340a328 100644 --- a/cibuildwheel/resources/constraints-python310.txt +++ b/cibuildwheel/resources/constraints-python310.txt @@ -18,6 +18,8 @@ packaging==25.0 # via # build # delocate +patchelf==0.17.2.4 + # via -r cibuildwheel/resources/constraints.in pip==25.3 # via -r cibuildwheel/resources/constraints.in platformdirs==4.5.1 diff --git a/cibuildwheel/resources/constraints-python311.txt b/cibuildwheel/resources/constraints-python311.txt index c3b366c6b..f2ab400ce 100644 --- a/cibuildwheel/resources/constraints-python311.txt +++ b/cibuildwheel/resources/constraints-python311.txt @@ -16,6 +16,8 @@ packaging==25.0 # via # build # delocate +patchelf==0.17.2.4 + # via -r cibuildwheel/resources/constraints.in pip==25.3 # via -r cibuildwheel/resources/constraints.in platformdirs==4.5.1 diff --git a/cibuildwheel/resources/constraints-python312.txt b/cibuildwheel/resources/constraints-python312.txt index c3b366c6b..f2ab400ce 100644 --- a/cibuildwheel/resources/constraints-python312.txt +++ b/cibuildwheel/resources/constraints-python312.txt @@ -16,6 +16,8 @@ packaging==25.0 # via # build # delocate +patchelf==0.17.2.4 + # via -r cibuildwheel/resources/constraints.in pip==25.3 # via -r cibuildwheel/resources/constraints.in platformdirs==4.5.1 diff --git a/cibuildwheel/resources/constraints-python313.txt b/cibuildwheel/resources/constraints-python313.txt index c3b366c6b..f2ab400ce 100644 --- a/cibuildwheel/resources/constraints-python313.txt +++ b/cibuildwheel/resources/constraints-python313.txt @@ -16,6 +16,8 @@ packaging==25.0 # via # build # delocate +patchelf==0.17.2.4 + # via -r cibuildwheel/resources/constraints.in pip==25.3 # via -r cibuildwheel/resources/constraints.in platformdirs==4.5.1 diff --git a/cibuildwheel/resources/constraints-python314.txt b/cibuildwheel/resources/constraints-python314.txt index c3b366c6b..f2ab400ce 100644 --- a/cibuildwheel/resources/constraints-python314.txt +++ b/cibuildwheel/resources/constraints-python314.txt @@ -16,6 +16,8 @@ packaging==25.0 # via # build # delocate +patchelf==0.17.2.4 + # via -r cibuildwheel/resources/constraints.in pip==25.3 # via -r cibuildwheel/resources/constraints.in platformdirs==4.5.1 diff --git a/cibuildwheel/resources/constraints-python38.txt b/cibuildwheel/resources/constraints-python38.txt index c7fe84056..c11f8eaa8 100644 --- a/cibuildwheel/resources/constraints-python38.txt +++ b/cibuildwheel/resources/constraints-python38.txt @@ -18,6 +18,8 @@ packaging==25.0 # via # build # delocate +patchelf==0.17.2.4 + # via -r cibuildwheel/resources/constraints.in pip==25.0.1 # via -r cibuildwheel/resources/constraints.in platformdirs==4.3.6 diff --git a/cibuildwheel/resources/constraints-python39.txt b/cibuildwheel/resources/constraints-python39.txt index 634899421..9b6f7e7fb 100644 --- a/cibuildwheel/resources/constraints-python39.txt +++ b/cibuildwheel/resources/constraints-python39.txt @@ -18,6 +18,8 @@ packaging==25.0 # via # build # delocate +patchelf==0.17.2.4 + # via -r cibuildwheel/resources/constraints.in pip==25.3 # via -r cibuildwheel/resources/constraints.in platformdirs==4.4.0 diff --git a/cibuildwheel/resources/constraints.in b/cibuildwheel/resources/constraints.in index 50bfabb6e..713c93ee2 100644 --- a/cibuildwheel/resources/constraints.in +++ b/cibuildwheel/resources/constraints.in @@ -1,4 +1,5 @@ pip build +patchelf delocate virtualenv diff --git a/cibuildwheel/resources/constraints.txt b/cibuildwheel/resources/constraints.txt index c3b366c6b..f2ab400ce 100644 --- a/cibuildwheel/resources/constraints.txt +++ b/cibuildwheel/resources/constraints.txt @@ -16,6 +16,8 @@ packaging==25.0 # via # build # delocate +patchelf==0.17.2.4 + # via -r cibuildwheel/resources/constraints.in pip==25.3 # via -r cibuildwheel/resources/constraints.in platformdirs==4.5.1 diff --git a/docs/options.md b/docs/options.md index c94cb0424..098bb6c36 100644 --- a/docs/options.md +++ b/docs/options.md @@ -893,8 +893,7 @@ Default: - on Linux: `'auditwheel repair -w {dest_dir} {wheel}'` - on macOS: `'delocate-wheel --require-archs {delocate_archs} -w {dest_dir} -v {wheel}'` -- on Android: There is no default command, but cibuildwheel will add `libc++` to the - wheel if anything links against it. Setting a command will replace this behavior. +- on Android: `'auditwheel repair --ldpaths {ldpaths} -w {dest_dir} {wheel}'` - on other platforms: `''` A shell command to repair a built wheel by copying external library dependencies into the wheel tree and relinking them. @@ -905,6 +904,7 @@ The following placeholders must be used inside the command and will be replaced - `{wheel}` for the absolute path to the built wheel - `{dest_dir}` for the absolute path of the directory where to create the repaired wheel - `{delocate_archs}` (macOS only) comma-separated list of architectures in the wheel. +- `{ldpaths}` (Android only) colon-separated list of directories to search for external libraries. You can use the `{package}` or `{project}` placeholders in your `repair-wheel-command` to refer to the package being built or the project root, respectively. diff --git a/pyproject.toml b/pyproject.toml index 6e8345586..642c6f62b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -45,8 +45,6 @@ dependencies = [ "filelock", "humanize", "packaging>=20.9", - # patchelf is used for Android - "patchelf; (sys_platform == 'linux' or sys_platform == 'darwin') and (platform_machine == 'x86_64' or platform_machine == 'arm64' or platform_machine == 'aarch64')", "platformdirs", "pyelftools>=0.29", "wheel>=0.33.6", From 2996c101cd488eda3d1310a1ac510cb405b51750 Mon Sep 17 00:00:00 2001 From: Malcolm Smith Date: Tue, 16 Dec 2025 17:56:19 +0000 Subject: [PATCH 02/14] Add auditwheel command to defaults --- bin/generate_schema.py | 2 ++ cibuildwheel/resources/defaults.toml | 4 ++++ 2 files changed, 6 insertions(+) diff --git a/bin/generate_schema.py b/bin/generate_schema.py index f6f6072bd..c1ac0af38 100755 --- a/bin/generate_schema.py +++ b/bin/generate_schema.py @@ -15,6 +15,8 @@ parser.add_argument("--schemastore", action="store_true", help="Generate schema_store version") args = parser.parse_args() +# The defaults in the schema are only used for documentation and IDE support. They +# should match the values in defaults.toml, which are used by cibuildwheel itself. starter = """ $schema: http://json-schema.org/draft-07/schema# $id: https://github.com/pypa/cibuildwheel/blob/main/cibuildwheel/resources/cibuildwheel.schema.json diff --git a/cibuildwheel/resources/defaults.toml b/cibuildwheel/resources/defaults.toml index 78895bf9a..60a41e183 100644 --- a/cibuildwheel/resources/defaults.toml +++ b/cibuildwheel/resources/defaults.toml @@ -1,3 +1,6 @@ +# These are the defaults used by cibuildwheel itself. They should match the values in +# generate_schema.py, which are used for documentation and IDE support. + [tool.cibuildwheel] build = "*" skip = "" @@ -60,6 +63,7 @@ repair-wheel-command = "delocate-wheel --require-archs {delocate_archs} -w {dest [tool.cibuildwheel.windows] [tool.cibuildwheel.android] +repair-wheel-command = "auditwheel repair --ldpaths {ldpaths} -w {dest_dir} {wheel}" [tool.cibuildwheel.ios] From f650f03c3647d1ce975a76472936fd528e8a297a Mon Sep 17 00:00:00 2001 From: Malcolm Smith Date: Tue, 16 Dec 2025 18:00:21 +0000 Subject: [PATCH 03/14] pkgconfig fixes --- cibuildwheel/platforms/android.py | 11 +++++++++-- cibuildwheel/resources/constraints-python310.txt | 2 ++ cibuildwheel/resources/constraints-python311.txt | 2 ++ cibuildwheel/resources/constraints-python312.txt | 2 ++ cibuildwheel/resources/constraints-python313.txt | 2 ++ cibuildwheel/resources/constraints-python314.txt | 2 ++ cibuildwheel/resources/constraints-python38.txt | 6 +++++- cibuildwheel/resources/constraints-python39.txt | 6 +++++- cibuildwheel/resources/constraints.in | 7 ++++++- cibuildwheel/resources/constraints.txt | 2 ++ 10 files changed, 37 insertions(+), 5 deletions(-) diff --git a/cibuildwheel/platforms/android.py b/cibuildwheel/platforms/android.py index 63eb3d770..fd7410099 100644 --- a/cibuildwheel/platforms/android.py +++ b/cibuildwheel/platforms/android.py @@ -169,6 +169,12 @@ def setup_target_python(config: PythonConfiguration, build_path: Path) -> Path: python_dir = build_path / "python" python_dir.mkdir() shutil.unpack_archive(python_tgz, python_dir) + + # Work around https://github.com/python/cpython/issues/138800. This can be removed + # once we've updated to Python versions that include the fix. + pc_path = python_dir / f"prefix/lib/pkgconfig/python-{config.version}.pc" + pc_path.write_text(pc_path.read_text().replace("$(BLDLIBRARY)", f"-lpython{config.version}")) + return python_dir @@ -227,6 +233,7 @@ def setup_env( tools = [ "auditwheel @ git+https://github.com/mhsmith/auditwheel@android", "patchelf", + "pkgconf", ] if build_frontend in {"build", "build[uv]"}: tools.append("build") @@ -408,7 +415,7 @@ def before_build(state: BuildState) -> None: shell_prepared( state.options.before_build, build_options=state.options, - env=state.build_env, + env=state.android_env, ) @@ -511,7 +518,7 @@ def test_wheel(state: BuildState, wheel: Path, *, build_frontend: str) -> None: shell_prepared( state.options.before_test, build_options=state.options, - env=state.build_env, + env=state.android_env, ) platform_args = ( diff --git a/cibuildwheel/resources/constraints-python310.txt b/cibuildwheel/resources/constraints-python310.txt index 94340a328..c9572c564 100644 --- a/cibuildwheel/resources/constraints-python310.txt +++ b/cibuildwheel/resources/constraints-python310.txt @@ -22,6 +22,8 @@ patchelf==0.17.2.4 # via -r cibuildwheel/resources/constraints.in pip==25.3 # via -r cibuildwheel/resources/constraints.in +pkgconf==2.5.1.post0 + # via -r cibuildwheel/resources/constraints.in platformdirs==4.5.1 # via virtualenv pyproject-hooks==1.2.0 diff --git a/cibuildwheel/resources/constraints-python311.txt b/cibuildwheel/resources/constraints-python311.txt index f2ab400ce..b7d84a988 100644 --- a/cibuildwheel/resources/constraints-python311.txt +++ b/cibuildwheel/resources/constraints-python311.txt @@ -20,6 +20,8 @@ patchelf==0.17.2.4 # via -r cibuildwheel/resources/constraints.in pip==25.3 # via -r cibuildwheel/resources/constraints.in +pkgconf==2.5.1.post0 + # via -r cibuildwheel/resources/constraints.in platformdirs==4.5.1 # via virtualenv pyproject-hooks==1.2.0 diff --git a/cibuildwheel/resources/constraints-python312.txt b/cibuildwheel/resources/constraints-python312.txt index f2ab400ce..b7d84a988 100644 --- a/cibuildwheel/resources/constraints-python312.txt +++ b/cibuildwheel/resources/constraints-python312.txt @@ -20,6 +20,8 @@ patchelf==0.17.2.4 # via -r cibuildwheel/resources/constraints.in pip==25.3 # via -r cibuildwheel/resources/constraints.in +pkgconf==2.5.1.post0 + # via -r cibuildwheel/resources/constraints.in platformdirs==4.5.1 # via virtualenv pyproject-hooks==1.2.0 diff --git a/cibuildwheel/resources/constraints-python313.txt b/cibuildwheel/resources/constraints-python313.txt index f2ab400ce..b7d84a988 100644 --- a/cibuildwheel/resources/constraints-python313.txt +++ b/cibuildwheel/resources/constraints-python313.txt @@ -20,6 +20,8 @@ patchelf==0.17.2.4 # via -r cibuildwheel/resources/constraints.in pip==25.3 # via -r cibuildwheel/resources/constraints.in +pkgconf==2.5.1.post0 + # via -r cibuildwheel/resources/constraints.in platformdirs==4.5.1 # via virtualenv pyproject-hooks==1.2.0 diff --git a/cibuildwheel/resources/constraints-python314.txt b/cibuildwheel/resources/constraints-python314.txt index f2ab400ce..b7d84a988 100644 --- a/cibuildwheel/resources/constraints-python314.txt +++ b/cibuildwheel/resources/constraints-python314.txt @@ -20,6 +20,8 @@ patchelf==0.17.2.4 # via -r cibuildwheel/resources/constraints.in pip==25.3 # via -r cibuildwheel/resources/constraints.in +pkgconf==2.5.1.post0 + # via -r cibuildwheel/resources/constraints.in platformdirs==4.5.1 # via virtualenv pyproject-hooks==1.2.0 diff --git a/cibuildwheel/resources/constraints-python38.txt b/cibuildwheel/resources/constraints-python38.txt index c11f8eaa8..f84244449 100644 --- a/cibuildwheel/resources/constraints-python38.txt +++ b/cibuildwheel/resources/constraints-python38.txt @@ -11,7 +11,9 @@ distlib==0.4.0 filelock==3.16.1 # via virtualenv importlib-metadata==8.5.0 - # via build + # via + # build + # pkgconf macholib==1.16.4 # via delocate packaging==25.0 @@ -22,6 +24,8 @@ patchelf==0.17.2.4 # via -r cibuildwheel/resources/constraints.in pip==25.0.1 # via -r cibuildwheel/resources/constraints.in +pkgconf==2.2.0.post0 + # via -r cibuildwheel/resources/constraints.in platformdirs==4.3.6 # via virtualenv pyproject-hooks==1.2.0 diff --git a/cibuildwheel/resources/constraints-python39.txt b/cibuildwheel/resources/constraints-python39.txt index 9b6f7e7fb..16ebf871b 100644 --- a/cibuildwheel/resources/constraints-python39.txt +++ b/cibuildwheel/resources/constraints-python39.txt @@ -11,7 +11,9 @@ distlib==0.4.0 filelock==3.19.1 # via virtualenv importlib-metadata==8.7.0 - # via build + # via + # build + # pkgconf macholib==1.16.4 # via delocate packaging==25.0 @@ -22,6 +24,8 @@ patchelf==0.17.2.4 # via -r cibuildwheel/resources/constraints.in pip==25.3 # via -r cibuildwheel/resources/constraints.in +pkgconf==2.4.3.post2 + # via -r cibuildwheel/resources/constraints.in platformdirs==4.4.0 # via virtualenv pyproject-hooks==1.2.0 diff --git a/cibuildwheel/resources/constraints.in b/cibuildwheel/resources/constraints.in index 713c93ee2..562df62cf 100644 --- a/cibuildwheel/resources/constraints.in +++ b/cibuildwheel/resources/constraints.in @@ -1,5 +1,10 @@ pip build +virtualenv + +# Android patchelf +pkgconf + +# macOS delocate -virtualenv diff --git a/cibuildwheel/resources/constraints.txt b/cibuildwheel/resources/constraints.txt index f2ab400ce..b7d84a988 100644 --- a/cibuildwheel/resources/constraints.txt +++ b/cibuildwheel/resources/constraints.txt @@ -20,6 +20,8 @@ patchelf==0.17.2.4 # via -r cibuildwheel/resources/constraints.in pip==25.3 # via -r cibuildwheel/resources/constraints.in +pkgconf==2.5.1.post0 + # via -r cibuildwheel/resources/constraints.in platformdirs==4.5.1 # via virtualenv pyproject-hooks==1.2.0 From 5f509e28959426633518e8ca532940d840e38e80 Mon Sep 17 00:00:00 2001 From: Malcolm Smith Date: Thu, 18 Dec 2025 00:31:54 +0000 Subject: [PATCH 04/14] Pre-import ctypes before monkey-patching in _cross_venv --- cibuildwheel/resources/_cross_venv.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/cibuildwheel/resources/_cross_venv.py b/cibuildwheel/resources/_cross_venv.py index 40dfaca5f..0c50dcce2 100644 --- a/cibuildwheel/resources/_cross_venv.py +++ b/cibuildwheel/resources/_cross_venv.py @@ -14,6 +14,9 @@ def initialize() -> None: if not (host_triplet := os.environ.get("CIBW_HOST_TRIPLET")): return + # Pre-import any modules which would fail to import after the monkey-patching. + import ctypes # noqa: F401, PLC0415 - uses get_config_var("LDLIBRARY") + # os ###################################################################### def cross_os_uname() -> os.uname_result: return os.uname_result( From 9d560aea34da8de0b053caa40ed27b0675faacf4 Mon Sep 17 00:00:00 2001 From: Malcolm Smith Date: Fri, 9 Jan 2026 17:09:27 +0000 Subject: [PATCH 05/14] Set PKG_CONFIG and PKG_CONFIG_RELOCATE_PATHS variables --- cibuildwheel/platforms/android.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/cibuildwheel/platforms/android.py b/cibuildwheel/platforms/android.py index fd7410099..68f3d918a 100644 --- a/cibuildwheel/platforms/android.py +++ b/cibuildwheel/platforms/android.py @@ -242,6 +242,12 @@ def setup_env( raise errors.FatalError(msg) call(*pip, "install", *tools, *constraint_flags(dependency_constraint), env=build_env) + # android-env.sh sets PKG_CONFIG="pkg-config --define-prefix", but some build + # systems can't handle arguments in that variable. Since we have a known version + # of pkgconf, it's safe to use PKG_CONFIG_RELOCATE_PATHS instead. + build_env["PKG_CONFIG"] = call("which", "pkgconf", env=build_env, capture_stdout=True).strip() + build_env["PKG_CONFIG_RELOCATE_PATHS"] = "1" + # Build-time requirements must be queried within android_env, because # `get_requires_for_build` can run arbitrary code in setup.py scripts, which may be # affected by the target platform. However, the requirements must be installed From 93053e7f795b009f25c0a3c16e8543b251e4f935 Mon Sep 17 00:00:00 2001 From: Malcolm Smith Date: Fri, 9 Jan 2026 17:18:40 +0000 Subject: [PATCH 06/14] Initial attempt at using mzakharo/android-gfortran --- .pre-commit-config.yaml | 2 +- cibuildwheel/platforms/android.py | 11 +++++++- .../resources/{ => android}/_cross_venv.pth | 0 .../resources/{ => android}/_cross_venv.py | 0 .../resources/android/fortran-entrypoint.sh | 9 ++++++ .../resources/android/fortran-shim.sh | 28 +++++++++++++++++++ .../resources/android/fortran.dockerfile | 15 ++++++++++ 7 files changed, 63 insertions(+), 2 deletions(-) rename cibuildwheel/resources/{ => android}/_cross_venv.pth (100%) rename cibuildwheel/resources/{ => android}/_cross_venv.py (100%) create mode 100755 cibuildwheel/resources/android/fortran-entrypoint.sh create mode 100755 cibuildwheel/resources/android/fortran-shim.sh create mode 100644 cibuildwheel/resources/android/fortran.dockerfile diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 9e0bbcd7b..5e0bec153 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -26,7 +26,7 @@ repos: - id: mypy name: mypy 3.11 on cibuildwheel/ args: ["--python-version=3.11"] - exclude: ^cibuildwheel/resources/_cross_venv.py$ # Requires Python 3.13 or later + exclude: ^cibuildwheel/resources/android/_cross_venv.py$ # Requires Python 3.13 or later additional_dependencies: &mypy-dependencies - bracex - build diff --git a/cibuildwheel/platforms/android.py b/cibuildwheel/platforms/android.py index 68f3d918a..ce97bf481 100644 --- a/cibuildwheel/platforms/android.py +++ b/cibuildwheel/platforms/android.py @@ -367,7 +367,7 @@ def setup_android_env( ) -> dict[str, str]: site_packages = next(venv_dir.glob("lib/python*/site-packages")) for suffix in ["pth", "py"]: - shutil.copy(resources.PATH / f"_cross_venv.{suffix}", site_packages) + shutil.copy(resources.PATH / f"android/_cross_venv.{suffix}", site_packages) sysconfigdata_path = Path( shutil.copy( @@ -407,6 +407,9 @@ def setup_android_env( for key in ["CFLAGS", "CXXFLAGS"]: android_env[key] += " " + opt + # Create shims which install additional build tools on first use. + setup_fortran(android_env) + # Format the environment so it can be pasted into a shell when debugging. for key, value in sorted(android_env.items()): if os.environ.get(key) != value: @@ -415,6 +418,12 @@ def setup_android_env( return android_env +def setup_fortran(env: dict[str, str]) -> None: + Path(f"{env['VIRTUAL_ENV']}/bin/gfortran").symlink_to( + resources.PATH / "android/fortran-shim.sh", + ) + + def before_build(state: BuildState) -> None: if state.options.before_build: log.step("Running before_build...") diff --git a/cibuildwheel/resources/_cross_venv.pth b/cibuildwheel/resources/android/_cross_venv.pth similarity index 100% rename from cibuildwheel/resources/_cross_venv.pth rename to cibuildwheel/resources/android/_cross_venv.pth diff --git a/cibuildwheel/resources/_cross_venv.py b/cibuildwheel/resources/android/_cross_venv.py similarity index 100% rename from cibuildwheel/resources/_cross_venv.py rename to cibuildwheel/resources/android/_cross_venv.py diff --git a/cibuildwheel/resources/android/fortran-entrypoint.sh b/cibuildwheel/resources/android/fortran-entrypoint.sh new file mode 100755 index 000000000..d9fb27d9a --- /dev/null +++ b/cibuildwheel/resources/android/fortran-entrypoint.sh @@ -0,0 +1,9 @@ +#!/bin/sh + +set -eu + +cwd="${1:?}" +shift + +cd "/host$cwd" +exec /root/*-linux-android-4.9/bin/*-linux-android-gfortran "$@" diff --git a/cibuildwheel/resources/android/fortran-shim.sh b/cibuildwheel/resources/android/fortran-shim.sh new file mode 100755 index 000000000..b1b8b3d01 --- /dev/null +++ b/cibuildwheel/resources/android/fortran-shim.sh @@ -0,0 +1,28 @@ +#!/bin/sh +# +# Pre-built copies of the Android Fortran compiler are only available for Linux x86_64, +# so we use Docker to allow it to run on macOS as well. +# +# TODO: only run docker build once, then replace this file with a different script. +set -eu + +cwd="$(pwd)" +cd "$(dirname "$(realpath "$0")")" + +arch="$(echo "$CIBW_HOST_TRIPLET" | sed 's/-.*//')" +tag="cibw-android-fortran-$arch" + +docker build \ + --platform linux/amd64 -f fortran.dockerfile -t "$tag" --build-arg arch="$arch" . + +# On macOS, Docker can only mount certain host directories by default. +if [ "$(uname)" = "Darwin" ]; then + mount_args="-v /Users:/host/Users -v /tmp:/host/tmp" +else + mount_args="-v /:/host" +fi + +# TODO mount /Users, /home, /private and /tmp if they exist, then pass working dir using +# -w. Then no entry point script is needed. +# shellcheck disable=SC2086 +docker run --rm --platform linux/amd64 $mount_args "$tag" "$cwd" "$@" diff --git a/cibuildwheel/resources/android/fortran.dockerfile b/cibuildwheel/resources/android/fortran.dockerfile new file mode 100644 index 000000000..f0abcd1b0 --- /dev/null +++ b/cibuildwheel/resources/android/fortran.dockerfile @@ -0,0 +1,15 @@ +FROM debian:trixie + +WORKDIR /root + +RUN apt-get update && \ + apt-get -y install bzip2 wget + +ARG arch +RUN if [ "${arch:?}" = "aarch64" ]; then arch="arm64"; fi && \ + filename="gcc-$arch-linux-x86_64.tar.bz2" && \ + wget "https://github.com/mzakharo/android-gfortran/releases/download/r21e/$filename" && \ + tar -xf "$filename" + +COPY fortran-entrypoint.sh ./ +ENTRYPOINT ["./fortran-entrypoint.sh"] From 926da78ea745e41845516e2a9bb3809d46a97008 Mon Sep 17 00:00:00 2001 From: Malcolm Smith Date: Sat, 10 Jan 2026 18:22:50 +0000 Subject: [PATCH 07/14] Switch to using termux/ndk-toolchain-clang-with-flang --- cibuildwheel/platforms/android.py | 14 +- .../resources/android/fortran-entrypoint.sh | 9 -- .../resources/android/fortran-shim.sh | 28 ---- .../resources/android/fortran.dockerfile | 15 -- .../resources/android/fortran_shim.py | 151 ++++++++++++++++++ 5 files changed, 162 insertions(+), 55 deletions(-) delete mode 100755 cibuildwheel/resources/android/fortran-entrypoint.sh delete mode 100755 cibuildwheel/resources/android/fortran-shim.sh delete mode 100644 cibuildwheel/resources/android/fortran.dockerfile create mode 100644 cibuildwheel/resources/android/fortran_shim.py diff --git a/cibuildwheel/platforms/android.py b/cibuildwheel/platforms/android.py index ce97bf481..a146a8e0e 100644 --- a/cibuildwheel/platforms/android.py +++ b/cibuildwheel/platforms/android.py @@ -4,6 +4,7 @@ import shlex import shutil import subprocess +import sys from dataclasses import dataclass from pathlib import Path from pprint import pprint @@ -419,9 +420,16 @@ def setup_android_env( def setup_fortran(env: dict[str, str]) -> None: - Path(f"{env['VIRTUAL_ENV']}/bin/gfortran").symlink_to( - resources.PATH / "android/fortran-shim.sh", - ) + # "flang-new" is the standard executable name for our current version of Flang. In + # future versions this will change to "flang" + # (https://blog.llvm.org/posts/2025-03-11-flang-new/). + shim_in = resources.PATH / "android/fortran_shim.py" + shim_out = Path(env["VIRTUAL_ENV"]) / "bin/flang-new" + + # The hashbang line runs the shim in cibuildwheel's own virtual environment, so it + # has access to utility functions for downloading and caching files. + shim_out.write_text(f"#!{sys.executable}\n\n" + shim_in.read_text()) + shim_out.chmod(0o755) def before_build(state: BuildState) -> None: diff --git a/cibuildwheel/resources/android/fortran-entrypoint.sh b/cibuildwheel/resources/android/fortran-entrypoint.sh deleted file mode 100755 index d9fb27d9a..000000000 --- a/cibuildwheel/resources/android/fortran-entrypoint.sh +++ /dev/null @@ -1,9 +0,0 @@ -#!/bin/sh - -set -eu - -cwd="${1:?}" -shift - -cd "/host$cwd" -exec /root/*-linux-android-4.9/bin/*-linux-android-gfortran "$@" diff --git a/cibuildwheel/resources/android/fortran-shim.sh b/cibuildwheel/resources/android/fortran-shim.sh deleted file mode 100755 index b1b8b3d01..000000000 --- a/cibuildwheel/resources/android/fortran-shim.sh +++ /dev/null @@ -1,28 +0,0 @@ -#!/bin/sh -# -# Pre-built copies of the Android Fortran compiler are only available for Linux x86_64, -# so we use Docker to allow it to run on macOS as well. -# -# TODO: only run docker build once, then replace this file with a different script. -set -eu - -cwd="$(pwd)" -cd "$(dirname "$(realpath "$0")")" - -arch="$(echo "$CIBW_HOST_TRIPLET" | sed 's/-.*//')" -tag="cibw-android-fortran-$arch" - -docker build \ - --platform linux/amd64 -f fortran.dockerfile -t "$tag" --build-arg arch="$arch" . - -# On macOS, Docker can only mount certain host directories by default. -if [ "$(uname)" = "Darwin" ]; then - mount_args="-v /Users:/host/Users -v /tmp:/host/tmp" -else - mount_args="-v /:/host" -fi - -# TODO mount /Users, /home, /private and /tmp if they exist, then pass working dir using -# -w. Then no entry point script is needed. -# shellcheck disable=SC2086 -docker run --rm --platform linux/amd64 $mount_args "$tag" "$cwd" "$@" diff --git a/cibuildwheel/resources/android/fortran.dockerfile b/cibuildwheel/resources/android/fortran.dockerfile deleted file mode 100644 index f0abcd1b0..000000000 --- a/cibuildwheel/resources/android/fortran.dockerfile +++ /dev/null @@ -1,15 +0,0 @@ -FROM debian:trixie - -WORKDIR /root - -RUN apt-get update && \ - apt-get -y install bzip2 wget - -ARG arch -RUN if [ "${arch:?}" = "aarch64" ]; then arch="arm64"; fi && \ - filename="gcc-$arch-linux-x86_64.tar.bz2" && \ - wget "https://github.com/mzakharo/android-gfortran/releases/download/r21e/$filename" && \ - tar -xf "$filename" - -COPY fortran-entrypoint.sh ./ -ENTRYPOINT ["./fortran-entrypoint.sh"] diff --git a/cibuildwheel/resources/android/fortran_shim.py b/cibuildwheel/resources/android/fortran_shim.py new file mode 100644 index 000000000..04f8792b5 --- /dev/null +++ b/cibuildwheel/resources/android/fortran_shim.py @@ -0,0 +1,151 @@ +# Based on +# https://github.com/kivy/python-for-android/blob/develop/pythonforandroid/recipes/fortran/__init__.py + +import os +import re +import shutil +import sys +from itertools import chain +from pathlib import Path + +from filelock import FileLock + +from cibuildwheel.util.file import CIBW_CACHE_PATH, download + +RELEASE_URL = "https://github.com/termux/ndk-toolchain-clang-with-flang/releases/download" +RELEASE_VERSION = "r27c" + +# The compiler is built for Linux x86_64, so we use Docker on macOS. +DOCKER_IMAGE = "debian:trixie" + + +def main() -> None: + cache_dir = CIBW_CACHE_PATH / f"flang-android-{RELEASE_VERSION}" + with FileLock(f"{cache_dir}.lock"): + if not cache_dir.exists(): + download_flang(cache_dir) + + flang_dir = Path(__file__).parents[2] / "flang" + with FileLock(f"{flang_dir}.lock"): + if not flang_dir.exists(): + setup_flang(cache_dir, flang_dir) + + run_flang(flang_dir) + + +def download_flang(cache_dir: Path) -> None: + tmp_dir = Path(f"{cache_dir}.tmp") + if tmp_dir.exists(): + shutil.rmtree(tmp_dir) + tmp_dir.mkdir(parents=True) + + for archive_name, src, dst in [ + ( + f"package-flang-{arch}.tar.bz2", + f"build-{arch}-install", + f"sysroot/usr/lib/{arch}-linux-android", + ) + for arch in ["aarch64", "x86_64"] + ] + [ + ("package-install.tar.bz2", "out/install/linux-x86/clang-dev", ""), + ("package-flang-host.tar.bz2", "build-host-install", ""), + ]: + archive_path = tmp_dir / archive_name + download(f"{RELEASE_URL}/{RELEASE_VERSION}/{archive_name}", archive_path) + shutil.unpack_archive(archive_path, tmp_dir) + archive_path.unlink() + + (tmp_dir / dst).mkdir(parents=True, exist_ok=True) + for src_path in (tmp_dir / src).iterdir(): + src_path.rename(tmp_dir / dst / src_path.name) + + tmp_dir.rename(cache_dir) + + +def setup_flang(cache_dir: Path, flang_dir: Path) -> None: + toolchain_dir = Path(os.environ["CC"]).parents[1] + ndk_dir = toolchain_dir.parents[3] + clang_ver_ndk = clang_ver(ndk_dir) + + clang_ver_cache = clang_ver(cache_dir) + if clang_ver_cache != clang_ver_ndk: + msg = f"Flang uses Clang {clang_ver_cache}, but NDK uses Clang {clang_ver_ndk}" + raise ValueError(msg) + + # Merge the Flang tree with the parts of the NDK it uses. + tmp_dir = Path(f"{flang_dir}.tmp") + if tmp_dir.exists(): + shutil.rmtree(tmp_dir) + tmp_dir.mkdir(parents=True) + + merge_path(cache_dir, tmp_dir) + merge_path(toolchain_dir, tmp_dir, f"lib/clang{clang_ver_ndk}/lib") + merge_path(toolchain_dir, tmp_dir, "sysroot") + + tmp_dir.rename(flang_dir) + + +def clang_ver(toolchain_dir: Path) -> str: + versions = [p.name for p in (toolchain_dir / "lib/clang").iterdir()] + assert len(versions) == 1 + return versions[0] + + +# The merged tree is more than 1 GB, so use symlinks to avoid copying. +def merge_path(src_dir: Path, dst_dir: Path, rel_path: str | None = None) -> None: + if rel_path is None: + for p in src_dir.iterdir(): + merge_path(src_dir, dst_dir, p.name) + return + + if not dst_dir.exists(): + dst_dir.mkdir() + elif dst_dir.is_dir(): + if dst_dir.is_symlink(): + old_src_dir = dst_dir.readlink() + dst_dir.unlink() + dst_dir.mkdir() + for p in old_src_dir.iterdir(): + (dst_dir / p.name).symlink_to(p) + else: + msg = f"{dst_dir} is not a directory" + raise ValueError(msg) + + prefix, sep, suffix = rel_path.partition("/") + if sep: + merge_path(src_dir / prefix, dst_dir / prefix, suffix) + else: + dst_path = dst_dir / rel_path + if dst_path.exists(): + merge_path(src_dir / rel_path, dst_dir / rel_path) + else: + dst_path.symlink_to(src_dir / rel_path) + + +def run_flang(flang_dir: Path) -> None: + match = re.fullmatch(r"(.+)-clang", os.environ["CC"]) + assert match is not None + args = [f"{flang_dir}/bin/flang-new", f"--target={match[1]}", *sys.argv[1:]] + + if sys.platform == "linux": + pass + elif sys.platform == "darwin": + args = [ + *["docker", "run"], + *chain.from_iterable( + # Docker on macOS only allows certain directories to be mounted as volumes, + # by default, but they include all the locations we're likely to need. + ["-v", f"{path}:{path}"] + for path in ["/private", "/Users", "/tmp"] + ), + *["-w", str(Path.cwd()), "--entrypoint", args[0], DOCKER_IMAGE, *args[1:]], + ] + else: + msg = f"unknown platform: {sys.platform}" + raise ValueError(msg) + + os.execvp(args[0], args) + + +if __name__ == "__main__": + main() From e996cd717319a6fb941d73321a2b24fe8d69434c Mon Sep 17 00:00:00 2001 From: Malcolm Smith Date: Sun, 11 Jan 2026 00:48:01 +0000 Subject: [PATCH 08/14] Move PKG_CONFIG variables from build_env to android_env --- cibuildwheel/platforms/android.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/cibuildwheel/platforms/android.py b/cibuildwheel/platforms/android.py index a146a8e0e..f0be50dc1 100644 --- a/cibuildwheel/platforms/android.py +++ b/cibuildwheel/platforms/android.py @@ -224,9 +224,6 @@ def setup_env( raise errors.FatalError(msg) call(command, "--version", env=build_env) - # Construct an altered environment which simulates running on Android. - android_env = setup_android_env(config, python_dir, venv_dir, build_env) - # Install build tools # TODO: use an official auditwheel version once # https://github.com/pypa/auditwheel/pull/643 has been released, and add it to the @@ -243,11 +240,8 @@ def setup_env( raise errors.FatalError(msg) call(*pip, "install", *tools, *constraint_flags(dependency_constraint), env=build_env) - # android-env.sh sets PKG_CONFIG="pkg-config --define-prefix", but some build - # systems can't handle arguments in that variable. Since we have a known version - # of pkgconf, it's safe to use PKG_CONFIG_RELOCATE_PATHS instead. - build_env["PKG_CONFIG"] = call("which", "pkgconf", env=build_env, capture_stdout=True).strip() - build_env["PKG_CONFIG_RELOCATE_PATHS"] = "1" + # Construct an altered environment which simulates running on Android. + android_env = setup_android_env(config, python_dir, venv_dir, build_env) # Build-time requirements must be queried within android_env, because # `get_requires_for_build` can run arbitrary code in setup.py scripts, which may be @@ -411,6 +405,12 @@ def setup_android_env( # Create shims which install additional build tools on first use. setup_fortran(android_env) + # `android.py env` returns PKG_CONFIG="pkg-config --define-prefix", but some build + # systems can't handle arguments in that variable. Since we have a known version + # of pkgconf, it's safe to use PKG_CONFIG_RELOCATE_PATHS instead. + android_env["PKG_CONFIG"] = call("which", "pkgconf", env=build_env, capture_stdout=True).strip() + android_env["PKG_CONFIG_RELOCATE_PATHS"] = "1" + # Format the environment so it can be pasted into a shell when debugging. for key, value in sorted(android_env.items()): if os.environ.get(key) != value: From bceed3c11d4a5e94561ed069de61f6d20edc7318 Mon Sep 17 00:00:00 2001 From: Malcolm Smith Date: Sun, 11 Jan 2026 00:54:02 +0000 Subject: [PATCH 09/14] Simplify flang installation --- cibuildwheel/platforms/android.py | 6 +- .../resources/android/fortran_shim.py | 120 ++++++------------ 2 files changed, 43 insertions(+), 83 deletions(-) diff --git a/cibuildwheel/platforms/android.py b/cibuildwheel/platforms/android.py index f0be50dc1..cc33b07f5 100644 --- a/cibuildwheel/platforms/android.py +++ b/cibuildwheel/platforms/android.py @@ -420,9 +420,8 @@ def setup_android_env( def setup_fortran(env: dict[str, str]) -> None: - # "flang-new" is the standard executable name for our current version of Flang. In - # future versions this will change to "flang" - # (https://blog.llvm.org/posts/2025-03-11-flang-new/). + # In case there's any autodetection based on the executable name, use the same name + # as the real executable (see fortran_shim.run_flang) shim_in = resources.PATH / "android/fortran_shim.py" shim_out = Path(env["VIRTUAL_ENV"]) / "bin/flang-new" @@ -430,6 +429,7 @@ def setup_fortran(env: dict[str, str]) -> None: # has access to utility functions for downloading and caching files. shim_out.write_text(f"#!{sys.executable}\n\n" + shim_in.read_text()) shim_out.chmod(0o755) + env["FC"] = str(shim_out) def before_build(state: BuildState) -> None: diff --git a/cibuildwheel/resources/android/fortran_shim.py b/cibuildwheel/resources/android/fortran_shim.py index 04f8792b5..00cb41422 100644 --- a/cibuildwheel/resources/android/fortran_shim.py +++ b/cibuildwheel/resources/android/fortran_shim.py @@ -1,6 +1,3 @@ -# Based on -# https://github.com/kivy/python-for-android/blob/develop/pythonforandroid/recipes/fortran/__init__.py - import os import re import shutil @@ -12,8 +9,11 @@ from cibuildwheel.util.file import CIBW_CACHE_PATH, download +# In the future we might pick a different Flang release depending on the NDK version, +# but so far all Python versions use the same NDK version, so there's no need. RELEASE_URL = "https://github.com/termux/ndk-toolchain-clang-with-flang/releases/download" RELEASE_VERSION = "r27c" +ARCHS = ["aarch64", "x86_64"] # The compiler is built for Linux x86_64, so we use Docker on macOS. DOCKER_IMAGE = "debian:trixie" @@ -25,12 +25,7 @@ def main() -> None: if not cache_dir.exists(): download_flang(cache_dir) - flang_dir = Path(__file__).parents[2] / "flang" - with FileLock(f"{flang_dir}.lock"): - if not flang_dir.exists(): - setup_flang(cache_dir, flang_dir) - - run_flang(flang_dir) + run_flang(cache_dir) def download_flang(cache_dir: Path) -> None: @@ -39,101 +34,66 @@ def download_flang(cache_dir: Path) -> None: shutil.rmtree(tmp_dir) tmp_dir.mkdir(parents=True) - for archive_name, src, dst in [ - ( - f"package-flang-{arch}.tar.bz2", - f"build-{arch}-install", - f"sysroot/usr/lib/{arch}-linux-android", - ) - for arch in ["aarch64", "x86_64"] - ] + [ - ("package-install.tar.bz2", "out/install/linux-x86/clang-dev", ""), - ("package-flang-host.tar.bz2", "build-host-install", ""), + for archive_name in [f"package-flang-{arch}.tar.bz2" for arch in ARCHS] + [ + "package-flang-host.tar.bz2", + "package-install.tar.bz2", ]: archive_path = tmp_dir / archive_name download(f"{RELEASE_URL}/{RELEASE_VERSION}/{archive_name}", archive_path) shutil.unpack_archive(archive_path, tmp_dir) archive_path.unlink() - (tmp_dir / dst).mkdir(parents=True, exist_ok=True) - for src_path in (tmp_dir / src).iterdir(): - src_path.rename(tmp_dir / dst / src_path.name) - - tmp_dir.rename(cache_dir) - + # Merge the extracted trees together, along with the necessary parts of the NDK. Based on + # https://github.com/kivy/python-for-android/blob/develop/pythonforandroid/recipes/fortran/__init__.py) + flang_toolchain = tmp_dir / "toolchain" + (tmp_dir / "out/install/linux-x86/clang-dev").rename(flang_toolchain) -def setup_flang(cache_dir: Path, flang_dir: Path) -> None: - toolchain_dir = Path(os.environ["CC"]).parents[1] - ndk_dir = toolchain_dir.parents[3] - clang_ver_ndk = clang_ver(ndk_dir) - - clang_ver_cache = clang_ver(cache_dir) - if clang_ver_cache != clang_ver_ndk: - msg = f"Flang uses Clang {clang_ver_cache}, but NDK uses Clang {clang_ver_ndk}" + ndk_toolchain = Path(os.environ["CC"]).parents[1] + if (clang_ver_flang := clang_ver(flang_toolchain)) != ( + clang_ver_ndk := clang_ver(ndk_toolchain) + ): + msg = f"Flang uses Clang {clang_ver_flang}, but NDK uses Clang {clang_ver_ndk}" raise ValueError(msg) - # Merge the Flang tree with the parts of the NDK it uses. - tmp_dir = Path(f"{flang_dir}.tmp") - if tmp_dir.exists(): - shutil.rmtree(tmp_dir) - tmp_dir.mkdir(parents=True) + clang_lib_path = f"lib/clang/{clang_ver_ndk}/lib" + shutil.rmtree(flang_toolchain / clang_lib_path) - merge_path(cache_dir, tmp_dir) - merge_path(toolchain_dir, tmp_dir, f"lib/clang{clang_ver_ndk}/lib") - merge_path(toolchain_dir, tmp_dir, "sysroot") + for src, dst in [ + (f"{tmp_dir}/build-{arch}-install", f"sysroot/usr/lib/{arch}-linux-android") + for arch in ARCHS + ] + [ + (f"{tmp_dir}/build-host-install", ""), + (f"{ndk_toolchain}/{clang_lib_path}", clang_lib_path), + (f"{ndk_toolchain}/sysroot", "sysroot"), + ]: + shutil.copytree(src, flang_toolchain / dst, symlinks=True, dirs_exist_ok=True) - tmp_dir.rename(flang_dir) + flang_toolchain.rename(cache_dir) + shutil.rmtree(tmp_dir) -def clang_ver(toolchain_dir: Path) -> str: - versions = [p.name for p in (toolchain_dir / "lib/clang").iterdir()] +def clang_ver(toolchain: Path) -> str: + versions = [p.name for p in (toolchain / "lib/clang").iterdir()] assert len(versions) == 1 return versions[0] -# The merged tree is more than 1 GB, so use symlinks to avoid copying. -def merge_path(src_dir: Path, dst_dir: Path, rel_path: str | None = None) -> None: - if rel_path is None: - for p in src_dir.iterdir(): - merge_path(src_dir, dst_dir, p.name) - return - - if not dst_dir.exists(): - dst_dir.mkdir() - elif dst_dir.is_dir(): - if dst_dir.is_symlink(): - old_src_dir = dst_dir.readlink() - dst_dir.unlink() - dst_dir.mkdir() - for p in old_src_dir.iterdir(): - (dst_dir / p.name).symlink_to(p) - else: - msg = f"{dst_dir} is not a directory" - raise ValueError(msg) - - prefix, sep, suffix = rel_path.partition("/") - if sep: - merge_path(src_dir / prefix, dst_dir / prefix, suffix) - else: - dst_path = dst_dir / rel_path - if dst_path.exists(): - merge_path(src_dir / rel_path, dst_dir / rel_path) - else: - dst_path.symlink_to(src_dir / rel_path) - - -def run_flang(flang_dir: Path) -> None: - match = re.fullmatch(r"(.+)-clang", os.environ["CC"]) +def run_flang(cache_dir: Path) -> None: + match = re.fullmatch(r".+/(.+)-clang", os.environ["CC"]) assert match is not None - args = [f"{flang_dir}/bin/flang-new", f"--target={match[1]}", *sys.argv[1:]] + target = match[1] + + # In a future Flang version the executable name will change to "flang" + # (https://blog.llvm.org/posts/2025-03-11-flang-new/). + args = [f"{cache_dir}/bin/flang-new", f"--target={target}", *sys.argv[1:]] if sys.platform == "linux": pass elif sys.platform == "darwin": args = [ - *["docker", "run"], + *["docker", "run", "--platform", "linux/amd64"], *chain.from_iterable( - # Docker on macOS only allows certain directories to be mounted as volumes, + # Docker on macOS only allows certain directories to be mounted as volumes # by default, but they include all the locations we're likely to need. ["-v", f"{path}:{path}"] for path in ["/private", "/Users", "/tmp"] From 488fd25a7c25f95a33b2e8cc1aa1b12bca749c6f Mon Sep 17 00:00:00 2001 From: Malcolm Smith Date: Sun, 11 Jan 2026 19:02:01 +0000 Subject: [PATCH 10/14] Add cross build files for NumPy --- cibuildwheel/platforms/android.py | 104 +++++++++++++++++++++--------- 1 file changed, 73 insertions(+), 31 deletions(-) diff --git a/cibuildwheel/platforms/android.py b/cibuildwheel/platforms/android.py index cc33b07f5..34da62660 100644 --- a/cibuildwheel/platforms/android.py +++ b/cibuildwheel/platforms/android.py @@ -22,6 +22,7 @@ from ..logger import log from ..options import BuildOptions, Options from ..selector import BuildSelector +from ..typing import PathOrStr from ..util import resources from ..util.cmd import call, shell from ..util.file import CIBW_CACHE_PATH, copy_test_sources, download, move_file @@ -35,6 +36,15 @@ "x86_64": "x86_64-linux-android", } +CROSS_BUILD_FILES = { + "numpy": [ + "numpy/_core/include/numpy/numpyconfig.h", + "numpy/_core/include/numpy/_numpyconfig.h", + "numpy/_core/lib/libnpymath.a", + "numpy/random/lib/libnpyrandom.a", + ] +} + def parse_identifier(identifier: str) -> tuple[str, str]: match = re.fullmatch(r"cp(\d)(\d+)-android_(.+)", identifier) @@ -130,6 +140,7 @@ def build(options: Options, tmp_path: Path) -> None: state = BuildState( config, build_options, build_path, python_dir, build_env, android_env ) + setup_cross_build_files(state) compatible_wheel = find_compatible_wheel(built_wheels, config.identifier) if compatible_wheel: @@ -143,7 +154,7 @@ def build(options: Options, tmp_path: Path) -> None: built_wheel = build_wheel(state) repaired_wheel = repair_wheel(state, built_wheel) - test_wheel(state, repaired_wheel, build_frontend=build_options.build_frontend.name) + test_wheel(state, repaired_wheel) output_wheel: Path | None = None if compatible_wheel is None: @@ -189,13 +200,7 @@ def setup_env( * android_env, which uses the environment while simulating running on Android. """ log.step("Setting up build environment...") - build_frontend = build_options.build_frontend.name - use_uv = build_frontend == "build[uv]" - uv_path = find_uv() - if use_uv and uv_path is None: - msg = "uv not found" - raise AssertionError(msg) - pip = ["pip"] if not use_uv else [str(uv_path), "pip"] + use_uv, pip = find_pip(build_options) # Create virtual environment python_exe = create_python_build_standalone_environment( @@ -233,7 +238,7 @@ def setup_env( "patchelf", "pkgconf", ] - if build_frontend in {"build", "build[uv]"}: + if build_options.build_frontend.name in {"build", "build[uv]"}: tools.append("build") else: msg = "Android requires the build frontend to be 'build'" @@ -241,7 +246,7 @@ def setup_env( call(*pip, "install", *tools, *constraint_flags(dependency_constraint), env=build_env) # Construct an altered environment which simulates running on Android. - android_env = setup_android_env(config, python_dir, venv_dir, build_env) + android_env = setup_android_env(config, python_dir, build_env) # Build-time requirements must be queried within android_env, because # `get_requires_for_build` can run arbitrary code in setup.py scripts, which may be @@ -358,9 +363,9 @@ def localized_vars( def setup_android_env( - config: PythonConfiguration, python_dir: Path, venv_dir: Path, build_env: dict[str, str] + config: PythonConfiguration, python_dir: Path, build_env: dict[str, str] ) -> dict[str, str]: - site_packages = next(venv_dir.glob("lib/python*/site-packages")) + site_packages = find_site_packages(build_env) for suffix in ["pth", "py"]: shutil.copy(resources.PATH / f"android/_cross_venv.{suffix}", site_packages) @@ -432,6 +437,59 @@ def setup_fortran(env: dict[str, str]) -> None: env["FC"] = str(shim_out) +# Although the build environment must be installed for the build platform, some packages +# contain platform-specific files which should be replaced with their Android +# equivalents. We do this using a similar technique to Pyodide: +# * https://github.com/pyodide/pyodide-build/blob/v0.30.2/pyodide_build/recipe/builder.py#L638 +# * https://github.com/pyodide/pyodide-recipes/blob/20250606/packages/numpy/meta.yaml#L28 +def setup_cross_build_files(state: BuildState) -> None: + _, pip = find_pip(state.options) + cbf_dir = state.build_path / "cross_build_files" + cbf_dir.mkdir() + + for requirement in call(*pip, "freeze", env=state.build_env, capture_stdout=True).splitlines(): + name, _, _ = requirement.strip().partition("==") + cross_build_files = CROSS_BUILD_FILES.get(name.lower(), []) + if cross_build_files: + pip_install_android(state, cbf_dir, "--no-deps", requirement) + for cbf in cross_build_files: + if (cbf_dir / cbf).exists(): + shutil.copy( + cbf_dir / cbf, + find_site_packages(state.build_env) / cbf, + ) + else: + log.warning(f"{cbf_dir / cbf} does not exist") + + +def pip_install_android(state: BuildState, target: Path, *args: PathOrStr) -> None: + use_uv, pip = find_pip(state.options) + call( + *pip, + "install", + "--only-binary=:all:", + *(["--python-platform", android_triplet(state.config.identifier)] if use_uv else []), + "--target", + target, + *args, + env=state.android_env, + ) + + +def find_site_packages(env: dict[str, str]) -> Path: + return next(Path(env["VIRTUAL_ENV"]).glob("lib/python*/site-packages")) + + +def find_pip(build_options: BuildOptions) -> tuple[bool, list[str]]: + use_uv = build_options.build_frontend.name == "build[uv]" + uv_path = find_uv() + if use_uv and uv_path is None: + msg = "uv not found" + raise AssertionError(msg) + pip = ["pip"] if not use_uv else [str(uv_path), "pip"] + return use_uv, pip + + def before_build(state: BuildState) -> None: if state.options.before_build: log.step("Running before_build...") @@ -516,19 +574,12 @@ def repair_wheel(state: BuildState, built_wheel: Path) -> Path: return repaired_wheel -def test_wheel(state: BuildState, wheel: Path, *, build_frontend: str) -> None: +def test_wheel(state: BuildState, wheel: Path) -> None: test_command = state.options.test_command if not (test_command and state.options.test_selector(state.config.identifier)): return log.step("Testing wheel...") - use_uv = build_frontend == "build[uv]" - uv_path = find_uv() - if use_uv and uv_path is None: - msg = "uv not found" - raise AssertionError(msg) - pip = ["pip"] if not use_uv else [str(uv_path), "pip"] - native_arch = arch_synonym(platform.machine(), platforms.native_platform(), "android") if state.config.arch != native_arch: log.warning( @@ -544,23 +595,14 @@ def test_wheel(state: BuildState, wheel: Path, *, build_frontend: str) -> None: env=state.android_env, ) - platform_args = ( - ["--python-platform", android_triplet(state.config.identifier)] if use_uv else [] - ) - # Install the wheel and test-requires. site_packages_dir = state.build_path / "site-packages" site_packages_dir.mkdir() - call( - *pip, - "install", - "--only-binary=:all:", - *platform_args, - "--target", + pip_install_android( + state, site_packages_dir, f"{wheel}{state.options.test_extras}", *state.options.test_requires, - env=state.android_env, ) # Copy test-sources. From 43310361c86b5bb9e4f0fa061838d0abb1a3b007 Mon Sep 17 00:00:00 2001 From: Malcolm Smith Date: Mon, 12 Jan 2026 12:19:17 +0000 Subject: [PATCH 11/14] Add ldpaths entry for libomp --- cibuildwheel/platforms/android.py | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/cibuildwheel/platforms/android.py b/cibuildwheel/platforms/android.py index 34da62660..deba02b74 100644 --- a/cibuildwheel/platforms/android.py +++ b/cibuildwheel/platforms/android.py @@ -538,13 +538,17 @@ def repair_wheel(state: BuildState, built_wheel: Path) -> Path: if state.options.repair_command: toolchain = Path(state.android_env["CC"]).parent.parent + triplet = android_triplet(state.config.identifier) ldpaths = ":".join( - # In the future, we may use this to implement PEP 725 by installing - # libraries in {state.python_dir}/prefix/lib or elsewhere, and adding that - # location to ldpaths. - [ - # For libc++_shared. - f"{toolchain}/sysroot/usr/lib/{state.android_env['CIBW_HOST_TRIPLET']}", + # Pass ldpaths to help auditwheel find compiler libraries. If we implement + # PEP 725 in the future to provide non-Python libraries, we'll need to add + # their location here. + str(next(Path(toolchain).glob(path))) + for path in [ + # libc++_shared + f"sysroot/usr/lib/{triplet}", + # libomp + f"lib/clang/*/lib/linux/{triplet.split('-')[0]}", ] ) shell( From cfda2c35c46e932abfac7f9469c595521634e191 Mon Sep 17 00:00:00 2001 From: Malcolm Smith Date: Thu, 29 Jan 2026 18:43:17 +0000 Subject: [PATCH 12/14] Add `--rm` to docker command line --- cibuildwheel/resources/android/fortran_shim.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cibuildwheel/resources/android/fortran_shim.py b/cibuildwheel/resources/android/fortran_shim.py index 00cb41422..ea2d3a472 100644 --- a/cibuildwheel/resources/android/fortran_shim.py +++ b/cibuildwheel/resources/android/fortran_shim.py @@ -91,7 +91,7 @@ def run_flang(cache_dir: Path) -> None: pass elif sys.platform == "darwin": args = [ - *["docker", "run", "--platform", "linux/amd64"], + *["docker", "run", "--rm", "--platform", "linux/amd64"], *chain.from_iterable( # Docker on macOS only allows certain directories to be mounted as volumes # by default, but they include all the locations we're likely to need. From d175c12437ce432c469517714c7bec0328b340f1 Mon Sep 17 00:00:00 2001 From: Malcolm Smith Date: Thu, 29 Jan 2026 19:07:30 +0000 Subject: [PATCH 13/14] Make Rust and Fortran shims consistent --- cibuildwheel/platforms/android.py | 9 ++------- cibuildwheel/resources/android/fortran_shim.py | 3 +++ .../resources/{_rust_shim.py => android/rust_shim.py} | 0 3 files changed, 5 insertions(+), 7 deletions(-) rename cibuildwheel/resources/{_rust_shim.py => android/rust_shim.py} (100%) diff --git a/cibuildwheel/platforms/android.py b/cibuildwheel/platforms/android.py index b5fec5a18..d1e1c71ad 100644 --- a/cibuildwheel/platforms/android.py +++ b/cibuildwheel/platforms/android.py @@ -5,7 +5,6 @@ import shutil import subprocess import sys -from collections.abc import MutableMapping from dataclasses import dataclass from pathlib import Path from pprint import pprint @@ -428,11 +427,7 @@ def setup_android_env( return android_env -def setup_rust( - config: PythonConfiguration, - python_dir: Path, - env: MutableMapping[str, str], -) -> None: +def setup_rust(config: PythonConfiguration, python_dir: Path, env: dict[str, str]) -> None: cargo_target = android_triplet(config.identifier) # CARGO_BUILD_TARGET is the variable used by Cargo and setuptools_rust @@ -452,7 +447,7 @@ def setup_rust( venv_bin = Path(env["VIRTUAL_ENV"]) / "bin" for tool in ["cargo", "rustup"]: shim_path = venv_bin / tool - shutil.copy(resources.PATH / "_rust_shim.py", shim_path) + shutil.copy(resources.PATH / "android/rust_shim.py", shim_path) shim_path.chmod(0o755) diff --git a/cibuildwheel/resources/android/fortran_shim.py b/cibuildwheel/resources/android/fortran_shim.py index ea2d3a472..62dba8bf0 100644 --- a/cibuildwheel/resources/android/fortran_shim.py +++ b/cibuildwheel/resources/android/fortran_shim.py @@ -1,3 +1,6 @@ +# This file intentionally has no hashbang line in the source: cibuildwheel will add it +# above this comment when the file is deployed. + import os import re import shutil diff --git a/cibuildwheel/resources/_rust_shim.py b/cibuildwheel/resources/android/rust_shim.py similarity index 100% rename from cibuildwheel/resources/_rust_shim.py rename to cibuildwheel/resources/android/rust_shim.py From f188500a73e86bc8da983745b8640e146ab7a23c Mon Sep 17 00:00:00 2001 From: Malcolm Smith Date: Thu, 29 Jan 2026 19:16:25 +0000 Subject: [PATCH 14/14] Update documentation --- docs/options.md | 2 +- docs/platforms.md | 5 ----- 2 files changed, 1 insertion(+), 6 deletions(-) diff --git a/docs/options.md b/docs/options.md index 098bb6c36..ff272345c 100644 --- a/docs/options.md +++ b/docs/options.md @@ -904,7 +904,7 @@ The following placeholders must be used inside the command and will be replaced - `{wheel}` for the absolute path to the built wheel - `{dest_dir}` for the absolute path of the directory where to create the repaired wheel - `{delocate_archs}` (macOS only) comma-separated list of architectures in the wheel. -- `{ldpaths}` (Android only) colon-separated list of directories to search for external libraries. +- `{ldpaths}` (Android only) colon-separated list of directories to search for external libraries. cibuildwheel will set this to include any necessary locations in the NDK. To add your own locations, use the `LD_LIBRARY_PATH` environment variable. You can use the `{package}` or `{project}` placeholders in your `repair-wheel-command` to refer to the package being built or the project root, respectively. diff --git a/docs/platforms.md b/docs/platforms.md index 6e46ec958..9574dde2f 100644 --- a/docs/platforms.md +++ b/docs/platforms.md @@ -214,11 +214,6 @@ minimum supported [API level](https://developer.android.com/tools/releases/platf for generated wheels. This will default to the minimum API level of the selected Python version. -If the [`repair-wheel-command`](options.md#repair-wheel-command) adds any libraries to -the wheel, then `ANDROID_API_LEVEL` must be at least 24. This is already the default -when building for Python 3.14 and later, but you may need to set it when building for -Python 3.13. - ### Build frontend support Android builds only support the `build` frontend. In principle, support for the