From 2d6724d5caf46f317fa6eeb4146592a971fc0c0c Mon Sep 17 00:00:00 2001 From: arasovic Date: Mon, 27 Jul 2026 00:45:44 +0300 Subject: [PATCH] fix(deps): resolve Pillow from piwheels on ARM32 --- contributors/emails/me@arasmehmet.com | 2 + pyproject.toml | 20 ++- tests/test_install_sh_uv_sources.py | 206 ++++++++++++++++++++++++++ tests/test_project_metadata.py | 114 +++++++++++++- uv.lock | 41 ++++- 5 files changed, 372 insertions(+), 11 deletions(-) create mode 100644 contributors/emails/me@arasmehmet.com create mode 100644 tests/test_install_sh_uv_sources.py diff --git a/contributors/emails/me@arasmehmet.com b/contributors/emails/me@arasmehmet.com new file mode 100644 index 0000000000000..a1ed75d74d5db --- /dev/null +++ b/contributors/emails/me@arasmehmet.com @@ -0,0 +1,2 @@ +arasovic +# PR #72134 diff --git a/pyproject.toml b/pyproject.toml index 52fd27046279c..df74b6d5938c6 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -371,7 +371,25 @@ override-dependencies = [ exclude-newer = "14 days" # h2: temporary exclude-newer exception for the CVE-2026-71554 (GHSA-6hr6-w5qg-qmwg, # request-smuggling) fix in 4.4.1, published 2026-08-03. Remove after 2026-08-17. -exclude-newer-package = { vercel = false, nemo-relay = false, huggingface_hub = false, h2 = false } +# piwheels omits upload timestamps, so uv cannot apply the age gate to Pillow. +# Pillow remains exact-pinned, index-scoped, and hash-locked in uv.lock. +exclude-newer-package = { vercel = false, nemo-relay = false, huggingface_hub = false, h2 = false, pillow = false } + +# PyPI publishes native aarch64 Pillow wheels, so only 32-bit ARM needs +# piwheels. piwheels publishes Pillow 12.3.0 for CPython 3.11 and 3.13 but +# not 3.12. Keep the index explicit so no other dependency can resolve from it. +# ARM32 Python 3.12 still uses the PyPI sdist and needs libjpeg/zlib headers. +# On matching versions, a missing piwheels wheel is a hard resolution failure; +# uv does not fall back to the sdist. See #72132. +[tool.uv.sources] +pillow = [ + { index = "piwheels", marker = "(platform_machine == 'armv6l' or platform_machine == 'armv7l') and (python_version == '3.11' or python_version == '3.13')" }, +] + +[[tool.uv.index]] +name = "piwheels" +url = "https://www.piwheels.org/simple" +explicit = true [tool.setuptools] # Top-level single-file modules (not packages). Without this, uv2nix's diff --git a/tests/test_install_sh_uv_sources.py b/tests/test_install_sh_uv_sources.py new file mode 100644 index 0000000000000..f3e16bd4a413b --- /dev/null +++ b/tests/test_install_sh_uv_sources.py @@ -0,0 +1,206 @@ +"""Behavioral coverage for the installer's uv fallback configuration.""" + +from functools import partial +from http.server import SimpleHTTPRequestHandler, ThreadingHTTPServer +import os +from pathlib import Path +import shutil +import subprocess +import sys +import threading +from zipfile import ZIP_DEFLATED, ZipFile + +import pytest + + +REPO_ROOT = Path(__file__).resolve().parents[1] +INSTALL_SH = REPO_ROOT / "scripts" / "install.sh" + + +class _QuietHandler(SimpleHTTPRequestHandler): + def log_message(self, _format, *_args): + pass + + +def _write_wheel(path: Path) -> None: + dist_info = "fallback_proof-1.0.0.dist-info" + with ZipFile(path, "w", ZIP_DEFLATED) as wheel: + wheel.writestr("fallback_proof/__init__.py", '__version__ = "1.0.0"\n') + wheel.writestr( + f"{dist_info}/METADATA", + "Metadata-Version: 2.1\nName: fallback-proof\nVersion: 1.0.0\n", + ) + wheel.writestr( + f"{dist_info}/WHEEL", + "Wheel-Version: 1.0\n" + "Generator: hermes-test\n" + "Root-Is-Purelib: true\n" + "Tag: py3-none-any\n", + ) + wheel.writestr(f"{dist_info}/RECORD", "") + + +def _write_uv_wrapper(path: Path) -> None: + """Fail Tier 0, then delegate fallback resolution to real uv as a dry run.""" + path.write_text( + """#!/bin/sh +printf 'UV_NO_CONFIG=%s UV_NO_SOURCES=%s %s\\n' \\ + "${UV_NO_CONFIG:-}" "${UV_NO_SOURCES:-}" "$*" >> "$UV_WRAPPER_LOG" +if [ "$1" = "sync" ]; then + exit 42 +fi +if [ "$1" = "pip" ] && [ "$2" = "install" ]; then + "$REAL_UV" "$@" --dry-run + status=$? + printf 'pip-status=%s\\n' "$status" >> "$UV_WRAPPER_LOG" + exit "$status" +fi +exec "$REAL_UV" "$@" +""", + encoding="utf-8", + ) + path.chmod(0o755) + + +def _run_python_deps_stage( + *, project: Path, hermes_home: Path, env: dict[str, str] +) -> subprocess.CompletedProcess[str]: + return subprocess.run( + [ + "bash", + str(INSTALL_SH), + "--stage", + "python-deps", + "--dir", + str(project), + "--hermes-home", + str(hermes_home), + ], + env=env, + capture_output=True, + text=True, + timeout=90, + ) + + +@pytest.mark.linux_only +def test_installer_fallback_uses_project_sources_with_uv_no_config(tmp_path): + """The real python-deps fallback must retain package-scoped uv sources.""" + uv = shutil.which("uv") + assert uv is not None, "uv must be available for installer integration tests" + + index_root = tmp_path / "index" + package_index = index_root / "simple" / "fallback-proof" + package_index.mkdir(parents=True) + wheel_name = "fallback_proof-1.0.0-py3-none-any.whl" + _write_wheel(package_index / wheel_name) + (package_index / "index.html").write_text( + f'{wheel_name}\n', encoding="utf-8" + ) + (index_root / "empty").mkdir() + + handler = partial(_QuietHandler, directory=str(index_root)) + server = ThreadingHTTPServer(("127.0.0.1", 0), handler) + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + + try: + base_url = f"http://127.0.0.1:{server.server_port}" + project = tmp_path / "project" + project.mkdir() + (project / "pyproject.toml").write_text( + f""" +[project] +name = "fallback-fixture" +version = "0.1.0" +requires-python = ">=3.11" +dependencies = ["fallback-proof==1.0.0"] + +[project.optional-dependencies] +all = [] + +[tool.uv.sources] +fallback-proof = {{ index = "fixture" }} + +[[tool.uv.index]] +name = "fixture" +url = "{base_url}/simple" +explicit = true +""".strip() + + "\n", + encoding="utf-8", + ) + # Presence enters Tier 0; the wrapper forces that locked sync to fail so + # install.sh must execute its real uv-pip recovery tier. + (project / "uv.lock").write_text("", encoding="utf-8") + + venv = project / "venv" + subprocess.run( + [uv, "venv", str(venv), "--python", sys.executable], + check=True, + capture_output=True, + text=True, + ) + + hermes_home = tmp_path / "hermes-home" + managed_bin = hermes_home / "bin" + managed_bin.mkdir(parents=True) + wrapper = managed_bin / "uv" + _write_uv_wrapper(wrapper) + wrapper_log = tmp_path / "uv-wrapper.log" + + tool_bin = tmp_path / "tool-bin" + tool_bin.mkdir() + dpkg = tool_bin / "dpkg" + dpkg.write_text("#!/bin/sh\nexit 0\n", encoding="utf-8") + dpkg.chmod(0o755) + + env = { + key: value + for key, value in os.environ.items() + if not key.startswith("UV_") + and key not in {"CONDA_DEFAULT_ENV", "CONDA_PREFIX", "VIRTUAL_ENV"} + } + env.update({ + "HOME": str(tmp_path / "home"), + "PATH": os.pathsep.join([ + str(tool_bin), + str(Path(sys.executable).parent), + env["PATH"], + ]), + "REAL_UV": uv, + "UV_CACHE_DIR": str(tmp_path / "cache"), + "UV_DEFAULT_INDEX": f"{base_url}/empty", + "UV_WRAPPER_LOG": str(wrapper_log), + }) + + negative_env = {**env, "UV_NO_SOURCES": "1"} + negative = _run_python_deps_stage( + project=project, hermes_home=hermes_home, env=negative_env + ) + negative_calls = wrapper_log.read_text(encoding="utf-8") + negative_statuses = [ + line.removeprefix("pip-status=") + for line in negative_calls.splitlines() + if line.startswith("pip-status=") + ] + + assert negative.returncode != 0 + assert "UV_NO_CONFIG=1 UV_NO_SOURCES=1 pip install -e .[all]" in negative_calls + assert negative_statuses + assert all(status != "0" for status in negative_statuses) + + wrapper_log.write_text("", encoding="utf-8") + result = _run_python_deps_stage( + project=project, hermes_home=hermes_home, env=env + ) + calls = wrapper_log.read_text(encoding="utf-8") + + assert result.returncode == 0, result.stdout + result.stderr + assert "UV_NO_CONFIG=1 UV_NO_SOURCES= sync --extra all --locked" in calls + assert "UV_NO_CONFIG=1 UV_NO_SOURCES= pip install -e .[all]" in calls + assert "pip-status=0" in calls + finally: + server.shutdown() + server.server_close() + thread.join(timeout=5) diff --git a/tests/test_project_metadata.py b/tests/test_project_metadata.py index 2b2ae47a6fd02..5fb18200e5202 100644 --- a/tests/test_project_metadata.py +++ b/tests/test_project_metadata.py @@ -3,11 +3,119 @@ from pathlib import Path import tomllib -def _load_optional_dependencies(): +from packaging.markers import Marker, default_environment + + +def _load_pyproject(): pyproject_path = Path(__file__).resolve().parents[1] / "pyproject.toml" with pyproject_path.open("rb") as handle: - project = tomllib.load(handle)["project"] - return project["optional-dependencies"] + return tomllib.load(handle) + + +def _load_optional_dependencies(): + return _load_pyproject()["project"]["optional-dependencies"] + + +def test_pillow_source_is_explicit_and_scoped_to_available_arm32_wheels(): + """Keep piwheels limited to Pillow where it publishes compatible wheels.""" + data = _load_pyproject() + uv_config = data["tool"]["uv"] + pillow_sources = uv_config["sources"]["pillow"] + piwheels = next( + index for index in uv_config["index"] if index["name"] == "piwheels" + ) + + assert piwheels["url"].rstrip("/") == "https://www.piwheels.org/simple" + assert piwheels["explicit"] is True + assert uv_config["exclude-newer-package"]["pillow"] is False + piwheels_source_keys = { + name + for name, sources in uv_config["sources"].items() + for source in (sources if isinstance(sources, list) else [sources]) + if source.get("index") == "piwheels" + } + assert piwheels_source_keys == {"pillow"} + assert len(pillow_sources) == 1 + assert pillow_sources[0]["index"] == "piwheels" + + marker = Marker(pillow_sources[0]["marker"]) + environment = default_environment() + supported_machines = {"armv6l", "armv7l"} + supported_pythons = {"3.11", "3.13"} + for machine in ("armv6l", "armv7l", "aarch64", "arm64", "x86_64"): + for python_version in ("3.11", "3.12", "3.13"): + environment["platform_machine"] = machine + environment["python_version"] = python_version + environment["python_full_version"] = f"{python_version}.0" + expected = ( + machine in supported_machines and python_version in supported_pythons + ) + assert marker.evaluate(environment) is expected + + +def test_arm32_pillow_wheels_are_hash_locked(): + """The trusted ARM32 artifacts must stay inside uv's SHA256 lock chain.""" + lock_path = Path(__file__).resolve().parents[1] / "uv.lock" + with lock_path.open("rb") as handle: + packages = tomllib.load(handle)["package"] + + pillow = next( + package + for package in packages + if package["name"] == "pillow" + and package["source"].get("registry", "").rstrip("/") + == "https://www.piwheels.org/simple" + ) + wheels = pillow["wheels"] + + assert any(wheel["url"].endswith("linux_armv6l.whl") for wheel in wheels) + assert any(wheel["url"].endswith("linux_armv7l.whl") for wheel in wheels) + assert all(wheel["hash"].startswith("sha256:") for wheel in wheels) + + +def test_locked_pillow_edges_preserve_expected_source_routing(): + """Hermes must route Pillow to the intended source for each platform.""" + lock_path = Path(__file__).resolve().parents[1] / "uv.lock" + with lock_path.open("rb") as handle: + packages = tomllib.load(handle)["package"] + + assert any( + package["name"] == "pillow" + and package["source"].get("registry", "").rstrip("/") + == "https://pypi.org/simple" + for package in packages + ), "PyPI Pillow entry must remain for non-ARM32 platforms" + hermes = next(package for package in packages if package["name"] == "hermes-agent") + pillow_edges = [ + dependency + for dependency in hermes["dependencies"] + if dependency["name"] == "pillow" + ] + edges_by_registry = { + edge["source"].get("registry", "").rstrip("/"): edge for edge in pillow_edges + } + assert set(edges_by_registry) == { + "https://pypi.org/simple", + "https://www.piwheels.org/simple", + } + + pypi_marker = Marker(edges_by_registry["https://pypi.org/simple"]["marker"]) + piwheels_marker = Marker( + edges_by_registry["https://www.piwheels.org/simple"]["marker"] + ) + environment = default_environment() + + for machine in ("armv6l", "armv7l", "aarch64", "arm64", "x86_64"): + for python_version in ("3.11", "3.12", "3.13"): + environment["platform_machine"] = machine + environment["python_version"] = python_version + environment["python_full_version"] = f"{python_version}.0" + use_piwheels = machine in {"armv6l", "armv7l"} and python_version in { + "3.11", + "3.13", + } + assert pypi_marker.evaluate(environment) is not use_piwheels + assert piwheels_marker.evaluate(environment) is use_piwheels def _load_package_data(): diff --git a/uv.lock b/uv.lock index e3733e6476c6e..8e1f5da929c73 100644 --- a/uv.lock +++ b/uv.lock @@ -2,17 +2,20 @@ version = 1 revision = 3 requires-python = ">=3.11, <3.14" resolution-markers = [ - "python_full_version >= '3.13'", + "python_full_version >= '3.13' and platform_machine != 'armv6l' and platform_machine != 'armv7l'", + "(python_full_version >= '3.13' and platform_machine == 'armv6l') or (python_full_version >= '3.13' and platform_machine == 'armv7l')", "python_full_version == '3.12.*'", - "python_full_version < '3.12'", + "python_full_version < '3.12' and platform_machine != 'armv6l' and platform_machine != 'armv7l'", + "(python_full_version < '3.12' and platform_machine == 'armv6l') or (python_full_version < '3.12' and platform_machine == 'armv7l')", ] [options] -exclude-newer = "2026-07-25T17:35:33.789835356Z" +exclude-newer = "2026-07-27T02:05:12.469831Z" exclude-newer-span = "P14D" [options.exclude-newer-package] vercel = false +pillow = false nemo-relay = false huggingface-hub = false h2 = false @@ -1576,7 +1579,8 @@ dependencies = [ { name = "openai" }, { name = "packaging" }, { name = "pathspec" }, - { name = "pillow" }, + { name = "pillow", version = "12.3.0", source = { registry = "https://pypi.org/simple" }, marker = "(python_full_version == '3.12.*' and platform_machine == 'armv6l') or (python_full_version == '3.12.*' and platform_machine == 'armv7l') or (platform_machine != 'armv6l' and platform_machine != 'armv7l')" }, + { name = "pillow", version = "12.3.0", source = { registry = "https://www.piwheels.org/simple" }, marker = "(python_full_version != '3.12.*' and platform_machine == 'armv6l') or (python_full_version != '3.12.*' and platform_machine == 'armv7l')" }, { name = "prompt-toolkit" }, { name = "psutil" }, { name = "ptyprocess", marker = "sys_platform != 'win32'" }, @@ -1887,7 +1891,8 @@ requires-dist = [ { name = "packaging", specifier = "==26.0" }, { name = "parallel-web", marker = "extra == 'parallel-web'", specifier = "==0.4.2" }, { name = "pathspec", specifier = "==1.1.1" }, - { name = "pillow", specifier = "==12.3.0" }, + { name = "pillow", marker = "(python_full_version < '3.11' and platform_machine == 'armv6l') or (python_full_version == '3.12.*' and platform_machine == 'armv6l') or (python_full_version >= '3.14' and platform_machine == 'armv6l') or (python_full_version < '3.11' and platform_machine == 'armv7l') or (python_full_version == '3.12.*' and platform_machine == 'armv7l') or (python_full_version >= '3.14' and platform_machine == 'armv7l') or (platform_machine != 'armv6l' and platform_machine != 'armv7l')", specifier = "==12.3.0" }, + { name = "pillow", marker = "(python_full_version == '3.11.*' and platform_machine == 'armv6l') or (python_full_version == '3.13.*' and platform_machine == 'armv6l') or (python_full_version == '3.11.*' and platform_machine == 'armv7l') or (python_full_version == '3.13.*' and platform_machine == 'armv7l')", specifier = "==12.3.0", index = "https://www.piwheels.org/simple" }, { name = "prompt-toolkit", specifier = "==3.0.52" }, { name = "psutil", specifier = "==7.2.2" }, { name = "ptyprocess", marker = "sys_platform != 'win32'", specifier = ">=0.7.0,<1" }, @@ -3065,6 +3070,11 @@ wheels = [ name = "pillow" version = "12.3.0" source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "python_full_version >= '3.13' and platform_machine != 'armv6l' and platform_machine != 'armv7l'", + "python_full_version == '3.12.*'", + "python_full_version < '3.12' and platform_machine != 'armv6l' and platform_machine != 'armv7l'", +] sdist = { url = "https://files.pythonhosted.org/packages/1c/3d/bb7fca845737cf9d7dbde16ed1843984665ff2e0a518f5db43e77ec540b9/pillow-12.3.0.tar.gz", hash = "sha256:3b8182a766685eaa002637e28b4ec8d6b18819a0c71f579bf0dbaa5830297cce", size = 47025035, upload-time = "2026-07-01T11:56:38.965Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/fb/c8/0a78b0e02d7ac54bc03e5321c9220da52f0c2ea83b21f7c40e7f3169c502/pillow-12.3.0-cp311-cp311-macosx_10_10_x86_64.whl", hash = "sha256:00808c5e14ef63ac5161091d242999076604ff74b883423a11e5d7bbb38bf756", size = 5392415, upload-time = "2026-07-01T11:53:47.162Z" }, @@ -3104,6 +3114,21 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/36/54/0169bc772ec491108b62f644f8ecf1fe5d8ae5ebafde2ee2142210166903/pillow-12.3.0-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:04f01d28a6aaff387bf842a13be313df23ba0597a44f1a976c9feb3c6ff4711a", size = 7231786, upload-time = "2026-07-01T11:56:35.046Z" }, ] +[[package]] +name = "pillow" +version = "12.3.0" +source = { registry = "https://www.piwheels.org/simple" } +resolution-markers = [ + "(python_full_version >= '3.13' and platform_machine == 'armv6l') or (python_full_version >= '3.13' and platform_machine == 'armv7l')", + "(python_full_version < '3.12' and platform_machine == 'armv6l') or (python_full_version < '3.12' and platform_machine == 'armv7l')", +] +wheels = [ + { url = "https://www.piwheels.org/simple/pillow/pillow-12.3.0-cp311-cp311-linux_armv6l.whl", hash = "sha256:402669e2c8551ae30ffafa9bb013d8797d7cd7132dd8ed5db14edb6f303c5d2a" }, + { url = "https://www.piwheels.org/simple/pillow/pillow-12.3.0-cp311-cp311-linux_armv7l.whl", hash = "sha256:402669e2c8551ae30ffafa9bb013d8797d7cd7132dd8ed5db14edb6f303c5d2a" }, + { url = "https://www.piwheels.org/simple/pillow/pillow-12.3.0-cp313-cp313-linux_armv6l.whl", hash = "sha256:d8b2b201942a627bca287184e7330d3ccbf4a6cce11ccc6ca8d2fb3bd1b08dff" }, + { url = "https://www.piwheels.org/simple/pillow/pillow-12.3.0-cp313-cp313-linux_armv7l.whl", hash = "sha256:d8b2b201942a627bca287184e7330d3ccbf4a6cce11ccc6ca8d2fb3bd1b08dff" }, +] + [[package]] name = "pluggy" version = "1.6.0" @@ -3996,7 +4021,8 @@ name = "scipy" version = "1.17.1" source = { registry = "https://pypi.org/simple" } resolution-markers = [ - "python_full_version < '3.12'", + "python_full_version < '3.12' and platform_machine != 'armv6l' and platform_machine != 'armv7l'", + "(python_full_version < '3.12' and platform_machine == 'armv6l') or (python_full_version < '3.12' and platform_machine == 'armv7l')", ] dependencies = [ { name = "numpy", marker = "python_full_version < '3.12'" }, @@ -4050,7 +4076,8 @@ name = "scipy" version = "1.18.0" source = { registry = "https://pypi.org/simple" } resolution-markers = [ - "python_full_version >= '3.13'", + "python_full_version >= '3.13' and platform_machine != 'armv6l' and platform_machine != 'armv7l'", + "(python_full_version >= '3.13' and platform_machine == 'armv6l') or (python_full_version >= '3.13' and platform_machine == 'armv7l')", "python_full_version == '3.12.*'", ] dependencies = [