feat: free-threaded Python for macOS, MACOSX_DEPLOYMENT_TARGET updates (#1854)

* Update dependencies

* feat: macos free-threading

Signed-off-by: Henry Schreiner <[email protected]>

* fix: set min target to 10.13 for 3.13 and floor it always

Signed-off-by: Henry Schreiner <[email protected]>

* fix: tests and corrected pypy mins

Signed-off-by: Henry Schreiner <[email protected]>

---------

Signed-off-by: Henry Schreiner <[email protected]>
Co-authored-by: cibuildwheel-bot[bot] <83877280+cibuildwheel-bot[bot]@users.noreply.github.com>
Co-authored-by: Henry Schreiner <[email protected]>
Co-authored-by: Matthieu Darbois <[email protected]>

Author: 3 people

.github/workflows/test.yml

@@ -82,6 +82,8 @@ jobs:
         output-dir: wheelhouse
       env:
         CIBW_ARCHS_MACOS: x86_64 universal2 arm64
+        CIBW_FREE_THREADED_SUPPORT: 1
+        CIBW_PRERELEASE_PYTHONS: 1
 
     - name: Run a sample build (GitHub Action, only)
       uses: ./

cibuildwheel/macos.py

@@ -33,6 +33,7 @@
     detect_ci_provider,
     download,
     find_compatible_wheel,
+    free_thread_enable_313,
     get_build_verbosity_extra_flags,
     get_pip_version,
     install_certifi_script,
@@ -115,12 +116,13 @@ def get_python_configurations(
     return python_configurations
 
 
-def install_cpython(tmp: Path, version: str, url: str) -> Path:
-    installation_path = Path(f"/Library/Frameworks/Python.framework/Versions/{version}")
+def install_cpython(tmp: Path, version: str, url: str, free_threading: bool) -> Path:
+    ft = "T" if free_threading else ""
+    installation_path = Path(f"/Library/Frameworks/Python{ft}.framework/Versions/{version}")
     with FileLock(CIBW_CACHE_PATH / f"cpython{version}.lock"):
         installed_system_packages = call("pkgutil", "--pkgs", capture_stdout=True).splitlines()
         # if this version of python isn't installed, get it from python.org and install
-        python_package_identifier = f"org.python.Python.PythonFramework-{version}"
+        python_package_identifier = f"org.python.Python.Python{ft}Framework-{version}"
         if python_package_identifier not in installed_system_packages:
             if detect_ci_provider() is None:
                 # if running locally, we don't want to install CPython with sudo
@@ -137,13 +139,22 @@ def install_cpython(tmp: Path, version: str, url: str) -> Path:
             # download the pkg
             download(url, pkg_path)
             # install
-            call("sudo", "installer", "-pkg", pkg_path, "-target", "/")
+            args = []
+            if version.startswith("3.13"):
+                # Python 3.13 is the first version to have a free-threading option
+                args += ["-applyChoiceChangesXML", str(free_thread_enable_313.resolve())]
+            call("sudo", "installer", "-pkg", pkg_path, *args, "-target", "/")
             pkg_path.unlink()
             env = os.environ.copy()
             env["PIP_DISABLE_PIP_VERSION_CHECK"] = "1"
-            call(installation_path / "bin" / "python3", install_certifi_script, env=env)
 
-    return installation_path / "bin" / "python3"
+            if free_threading:
+                call(installation_path / f"bin/python{version}t", "-m", "ensurepip", env=env)
+                call(installation_path / f"bin/python{version}t", install_certifi_script, env=env)
+            else:
+                call(installation_path / "bin/python3", install_certifi_script, env=env)
+
+    return installation_path / "bin" / (f"python{version}t" if free_threading else "python3")
 
 
 def install_pypy(tmp: Path, url: str) -> Path:
@@ -172,13 +183,19 @@ def setup_python(
     implementation_id = python_configuration.identifier.split("-")[0]
     log.step(f"Installing Python {implementation_id}...")
     if implementation_id.startswith("cp"):
-        base_python = install_cpython(tmp, python_configuration.version, python_configuration.url)
+        free_threading = "t-macos" in python_configuration.identifier
+        base_python = install_cpython(
+            tmp, python_configuration.version, python_configuration.url, free_threading
+        )
+
     elif implementation_id.startswith("pp"):
         base_python = install_pypy(tmp, python_configuration.url)
     else:
         msg = "Unknown Python implementation"
         raise ValueError(msg)
-    assert base_python.exists()
+    assert (
+        base_python.exists()
+    ), f"{base_python.name} not found, has {list(base_python.parent.iterdir())}"
 
     log.step("Setting up build environment...")
     venv_path = tmp / "venv"
@@ -244,8 +261,25 @@ def setup_python(
     # Set MACOSX_DEPLOYMENT_TARGET, if the user didn't set it.
     # For arm64, the minimal deployment target is 11.0.
     # On x86_64 (or universal2), use 10.9 as a default.
-    # PyPy defaults to 10.7, causing inconsistencies if it's left unset.
-    env.setdefault("MACOSX_DEPLOYMENT_TARGET", "11.0" if config_is_arm64 else "10.9")
+    # CPython 3.13 needs 10.13.
+    if config_is_arm64:
+        default_target = "11.0"
+    elif Version(python_configuration.version) >= Version("3.13"):
+        default_target = "10.13"
+    elif python_configuration.identifier.startswith("pp") and Version(
+        python_configuration.version
+    ) >= Version("3.9"):
+        default_target = "10.15"
+    else:
+        default_target = "10.9"
+    env.setdefault("MACOSX_DEPLOYMENT_TARGET", default_target)
+
+    # This is a floor, it can't be set lower than the default_target.
+    if Version(env["MACOSX_DEPLOYMENT_TARGET"]) < Version(default_target):
+        log.warning(
+            f"Bumping MACOSX_DEPLOYMENT_TARGET ({env['MACOSX_DEPLOYMENT_TARGET']}) to the minimum required ({default_target})."
+        )
+        env["MACOSX_DEPLOYMENT_TARGET"] = default_target
 
     if python_configuration.version not in {"3.6", "3.7"}:
         if config_is_arm64:

cibuildwheel/resources/build-platforms.toml

@@ -123,9 +123,12 @@ python_configurations = [
   { identifier = "cp312-macosx_x86_64", version = "3.12", url = "https://www.python.org/ftp/python/3.12.3/python-3.12.3-macos11.pkg" },
   { identifier = "cp312-macosx_arm64", version = "3.12", url = "https://www.python.org/ftp/python/3.12.3/python-3.12.3-macos11.pkg" },
   { identifier = "cp312-macosx_universal2", version = "3.12", url = "https://www.python.org/ftp/python/3.12.3/python-3.12.3-macos11.pkg" },
-  { identifier = "cp313-macosx_x86_64", version = "3.13", url = "https://www.python.org/ftp/python/3.13.0/python-3.13.0b1-macos11.pkg" },
-  { identifier = "cp313-macosx_arm64", version = "3.13", url = "https://www.python.org/ftp/python/3.13.0/python-3.13.0b1-macos11.pkg" },
-  { identifier = "cp313-macosx_universal2", version = "3.13", url = "https://www.python.org/ftp/python/3.13.0/python-3.13.0b1-macos11.pkg" },
+  { identifier = "cp313-macosx_x86_64", version = "3.13", url = "https://www.python.org/ftp/python/3.13.0/python-3.13.0b2-macos11.pkg" },
+  { identifier = "cp313-macosx_arm64", version = "3.13", url = "https://www.python.org/ftp/python/3.13.0/python-3.13.0b2-macos11.pkg" },
+  { identifier = "cp313-macosx_universal2", version = "3.13", url = "https://www.python.org/ftp/python/3.13.0/python-3.13.0b2-macos11.pkg" },
+  { identifier = "cp313t-macosx_x86_64", version = "3.13", url = "https://www.python.org/ftp/python/3.13.0/python-3.13.0b2-macos11.pkg" },
+  { identifier = "cp313t-macosx_arm64", version = "3.13", url = "https://www.python.org/ftp/python/3.13.0/python-3.13.0b2-macos11.pkg" },
+  { identifier = "cp313t-macosx_universal2", version = "3.13", url = "https://www.python.org/ftp/python/3.13.0/python-3.13.0b2-macos11.pkg" },
   { identifier = "pp37-macosx_x86_64", version = "3.7", url = "https://downloads.python.org/pypy/pypy3.7-v7.3.9-osx64.tar.bz2" },
   { identifier = "pp38-macosx_x86_64", version = "3.8", url = "https://downloads.python.org/pypy/pypy3.8-v7.3.11-macos_x86_64.tar.bz2" },
   { identifier = "pp38-macosx_arm64", version = "3.8", url = "https://downloads.python.org/pypy/pypy3.8-v7.3.11-macos_arm64.tar.bz2" },
@@ -149,18 +152,18 @@ python_configurations = [
   { identifier = "cp310-win_amd64", version = "3.10.11", arch = "64" },
   { identifier = "cp311-win32", version = "3.11.9", arch = "32" },
   { identifier = "cp311-win_amd64", version = "3.11.9", arch = "64" },
-  { identifier = "cp312-win32", version = "3.12.3", arch = "32" },
-  { identifier = "cp312-win_amd64", version = "3.12.3", arch = "64" },
-  { identifier = "cp313-win32", version = "3.13.0-b1", arch = "32" },
-  { identifier = "cp313t-win32", version = "3.13.0-b1", arch = "32" },
-  { identifier = "cp313-win_amd64", version = "3.13.0-b1", arch = "64" },
-  { identifier = "cp313t-win_amd64", version = "3.13.0-b1", arch = "64" },
+  { identifier = "cp312-win32", version = "3.12.4", arch = "32" },
+  { identifier = "cp312-win_amd64", version = "3.12.4", arch = "64" },
+  { identifier = "cp313-win32", version = "3.13.0-b2", arch = "32" },
+  { identifier = "cp313t-win32", version = "3.13.0-b2", arch = "32" },
+  { identifier = "cp313-win_amd64", version = "3.13.0-b2", arch = "64" },
+  { identifier = "cp313t-win_amd64", version = "3.13.0-b2", arch = "64" },
   { identifier = "cp39-win_arm64", version = "3.9.10", arch = "ARM64" },
   { identifier = "cp310-win_arm64", version = "3.10.11", arch = "ARM64" },
   { identifier = "cp311-win_arm64", version = "3.11.9", arch = "ARM64" },
-  { identifier = "cp312-win_arm64", version = "3.12.3", arch = "ARM64" },
-  { identifier = "cp313-win_arm64", version = "3.13.0-b1", arch = "ARM64" },
-  { identifier = "cp313t-win_arm64", version = "3.13.0-b1", arch = "ARM64" },
+  { identifier = "cp312-win_arm64", version = "3.12.4", arch = "ARM64" },
+  { identifier = "cp313-win_arm64", version = "3.13.0-b2", arch = "ARM64" },
+  { identifier = "cp313t-win_arm64", version = "3.13.0-b2", arch = "ARM64" },
   { identifier = "pp37-win_amd64", version = "3.7", arch = "64", url = "https://downloads.python.org/pypy/pypy3.7-v7.3.9-win64.zip" },
   { identifier = "pp38-win_amd64", version = "3.8", arch = "64", url = "https://downloads.python.org/pypy/pypy3.8-v7.3.11-win64.zip" },
   { identifier = "pp39-win_amd64", version = "3.9", arch = "64", url = "https://downloads.python.org/pypy/pypy3.9-v7.3.16-win64.zip" },

cibuildwheel/resources/free-threaded-enable-313.xml

@@ -0,0 +1,14 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
+<plist version="1.0">
+<array>
+        <dict>
+                <key>attributeSetting</key>
+                <integer>1</integer>
+                <key>choiceAttribute</key>
+                <string>selected</string>
+                <key>choiceIdentifier</key>
+                <string>org.python.Python.PythonTFramework-3.13</string>
+        </dict>
+</array>
+</plist>

cibuildwheel/resources/pinned_docker_images.cfg

@@ -1,29 +1,29 @@
 [x86_64]
 manylinux1 = quay.io/pypa/manylinux1_x86_64:2024-04-29-76807b8
 manylinux2010 = quay.io/pypa/manylinux2010_x86_64:2022-08-05-4535177
-manylinux2014 = quay.io/pypa/manylinux2014_x86_64:2024-06-03-e195670
+manylinux2014 = quay.io/pypa/manylinux2014_x86_64:2024-06-06-99f15a7
 manylinux_2_24 = quay.io/pypa/manylinux_2_24_x86_64:2022-12-26-0d38463
-manylinux_2_28 = quay.io/pypa/manylinux_2_28_x86_64:2024-06-03-e195670
-musllinux_1_1 = quay.io/pypa/musllinux_1_1_x86_64:2024-06-03-e195670
-musllinux_1_2 = quay.io/pypa/musllinux_1_2_x86_64:2024-06-03-e195670
+manylinux_2_28 = quay.io/pypa/manylinux_2_28_x86_64:2024-06-06-99f15a7
+musllinux_1_1 = quay.io/pypa/musllinux_1_1_x86_64:2024-06-06-99f15a7
+musllinux_1_2 = quay.io/pypa/musllinux_1_2_x86_64:2024-06-06-99f15a7
 
 [i686]
 manylinux1 = quay.io/pypa/manylinux1_i686:2024-04-29-76807b8
 manylinux2010 = quay.io/pypa/manylinux2010_i686:2022-08-05-4535177
-manylinux2014 = quay.io/pypa/manylinux2014_i686:2024-06-03-e195670
+manylinux2014 = quay.io/pypa/manylinux2014_i686:2024-06-06-99f15a7
 manylinux_2_24 = quay.io/pypa/manylinux_2_24_i686:2022-12-26-0d38463
-musllinux_1_1 = quay.io/pypa/musllinux_1_1_i686:2024-06-03-e195670
-musllinux_1_2 = quay.io/pypa/musllinux_1_2_i686:2024-06-03-e195670
+musllinux_1_1 = quay.io/pypa/musllinux_1_1_i686:2024-06-06-99f15a7
+musllinux_1_2 = quay.io/pypa/musllinux_1_2_i686:2024-06-06-99f15a7
 
 [pypy_x86_64]
 manylinux2010 = quay.io/pypa/manylinux2010_x86_64:2022-08-05-4535177
-manylinux2014 = quay.io/pypa/manylinux2014_x86_64:2024-06-03-e195670
+manylinux2014 = quay.io/pypa/manylinux2014_x86_64:2024-06-06-99f15a7
 manylinux_2_24 = quay.io/pypa/manylinux_2_24_x86_64:2022-12-26-0d38463
-manylinux_2_28 = quay.io/pypa/manylinux_2_28_x86_64:2024-06-03-e195670
+manylinux_2_28 = quay.io/pypa/manylinux_2_28_x86_64:2024-06-06-99f15a7
 
 [pypy_i686]
 manylinux2010 = quay.io/pypa/manylinux2010_i686:2022-08-05-4535177
-manylinux2014 = quay.io/pypa/manylinux2014_i686:2024-06-03-e195670
+manylinux2014 = quay.io/pypa/manylinux2014_i686:2024-06-06-99f15a7
 manylinux_2_24 = quay.io/pypa/manylinux_2_24_i686:2022-12-26-0d38463
 
 [aarch64]

cibuildwheel/util.py

@@ -60,6 +60,8 @@
 
 install_certifi_script: Final[Path] = resources_dir / "install_certifi.py"
 
+free_thread_enable_313: Final[Path] = resources_dir / "free-threaded-enable-313.xml"
+
 test_fail_cwd_file: Final[Path] = resources_dir / "testing_temp_dir_file.py"
 
 

test/test_macos_archs.py

@@ -14,7 +14,7 @@
     *utils.expected_wheels("spam", "0.1.0", machine_arch="arm64", include_universal2=True),
 }
 
-DEPLOYMENT_TARGET_TOO_LOW_WARNING = "[WARNING] MACOSX_DEPLOYMENT_TARGET is set to a lower value"
+DEPLOYMENT_TARGET_TOO_LOW_WARNING = "Bumping MACOSX_DEPLOYMENT_TARGET"
 
 
 def get_xcode_version() -> tuple[int, int]:

test/utils.py

@@ -137,17 +137,11 @@ def cibuildwheel_run(
     return wheels
 
 
-def _get_arm64_macosx_deployment_target(macosx_deployment_target: str) -> str:
+def _floor_macosx(*args: str) -> str:
     """
-    The first version of macOS that supports arm is 11.0. So the wheel tag
-    cannot contain an earlier deployment target, even if
-    MACOSX_DEPLOYMENT_TARGET sets it.
+    Make sure a deployment target is not less than some value.
     """
-    version_tuple = tuple(map(int, macosx_deployment_target.split(".")))
-    if version_tuple <= (11, 0):
-        return "11.0"
-    else:
-        return macosx_deployment_target
+    return max(args, key=lambda x: tuple(map(int, x.split("."))))
 
 
 def expected_wheels(
@@ -202,9 +196,8 @@ def expected_wheels(
             "cp311-cp311",
             "cp312-cp312",
             "cp313-cp313",
+            "cp313-cp313t",
         ]
-        if platform != "macos":
-            python_abi_tags.append("cp313-cp313t")
 
         if machine_arch in ["x86_64", "AMD64", "x86", "aarch64"]:
             python_abi_tags += [
@@ -223,6 +216,7 @@ def expected_wheels(
                 "cp311-cp311",
                 "cp312-cp312",
                 "cp313-cp313",
+                "cp313-cp313t",
                 "pp38-pypy38_pp73",
                 "pp39-pypy39_pp73",
                 "pp310-pypy310_pp73",
@@ -282,12 +276,19 @@ def expected_wheels(
 
         elif platform == "macos":
             if machine_arch == "arm64":
-                arm64_macosx_deployment_target = _get_arm64_macosx_deployment_target(
-                    macosx_deployment_target
-                )
-                platform_tags = [f'macosx_{arm64_macosx_deployment_target.replace(".", "_")}_arm64']
+                arm64_macosx = _floor_macosx(macosx_deployment_target, "11.0")
+                platform_tags = [f'macosx_{arm64_macosx.replace(".", "_")}_arm64']
             else:
-                platform_tags = [f'macosx_{macosx_deployment_target.replace(".", "_")}_x86_64']
+                if python_abi_tag.startswith("pp") and not python_abi_tag.startswith(
+                    ("pp37", "pp38")
+                ):
+                    pypy_macosx = _floor_macosx(macosx_deployment_target, "10.15")
+                    platform_tags = [f'macosx_{pypy_macosx.replace(".", "_")}_x86_64']
+                elif python_abi_tag.startswith("cp313"):
+                    pypy_macosx = _floor_macosx(macosx_deployment_target, "10.13")
+                    platform_tags = [f'macosx_{pypy_macosx.replace(".", "_")}_x86_64']
+                else:
+                    platform_tags = [f'macosx_{macosx_deployment_target.replace(".", "_")}_x86_64']
 
             if include_universal2:
                 platform_tags.append(