From f4ba656a41d6dea2ae46d80367959d85d80b421e Mon Sep 17 00:00:00 2001 From: Yong Wu Date: Tue, 18 Aug 2026 11:33:01 -0700 Subject: [PATCH 1/3] ci: use the CI image's packages and share a pip cache between jobs --- build_backend.py | 49 +++++++++++++ ci/bash.sh | 21 ++++++ docker/install/install_python_packages.sh | 4 +- scripts/check_requirements.py | 71 +++++++++++++++++++ scripts/setup_test_env.sh | 33 +++++++++ scripts/task_jit_run_tests_part1.sh | 2 +- scripts/task_jit_run_tests_part2.sh | 2 +- scripts/task_jit_run_tests_part3.sh | 2 +- scripts/task_jit_run_tests_part4.sh | 2 +- scripts/task_jit_run_tests_part5.sh | 2 +- ...ask_test_jit_cache_package_build_import.sh | 7 +- scripts/task_test_single_node_comm_kernels.sh | 2 +- scripts/test_utils.sh | 6 +- 13 files changed, 192 insertions(+), 11 deletions(-) create mode 100755 scripts/check_requirements.py diff --git a/build_backend.py b/build_backend.py index 2c6ad43f60a..a58c6530160 100644 --- a/build_backend.py +++ b/build_backend.py @@ -119,6 +119,15 @@ def _in_isolated_build_env() -> bool: return any(m in p for m in markers for p in paths) +def _no_pip_installs() -> bool: + """True when the environment is preprovisioned and the hooks must not install. + + Set by the CI scripts: they build with --no-build-isolation, which would + otherwise turn hook installs that isolation always swallowed into downloads. + """ + return os.environ.get("FLASHINFER_BUILD_NO_PIP") == "1" + + def _detect_cuda_major() -> int: """Best-effort detection of the CUDA major version on the host.""" try: @@ -567,6 +576,12 @@ def _ensure_nixl_wheel() -> None: except Exception: pass return + if _no_pip_installs(): + print( + "[BUILD_NVEP] FLASHINFER_BUILD_NO_PIP=1; no NIXL wheel to " + "pre-install, so the pre-flight probe will skip NIXL-EP" + ) + return cuda_major = _detect_cuda_major() wheel = f"nixl-cu{cuda_major}=={_NIXL_WHEEL_VERSION}" print(f"[BUILD_NVEP] pre-installing NIXL wheel --no-deps: {wheel}") @@ -602,6 +617,9 @@ def _ensure_nccl_floor() -> None: cuda_major = _detect_cuda_major() if cuda_major < 13: return # EP is CUDA-13-only; nothing to upgrade on cu12 hosts. + if _no_pip_installs(): + print("[BUILD_NVEP] FLASHINFER_BUILD_NO_PIP=1; leaving NCCL as installed") + return wheel = "nvidia-nccl-cu13>=2.30.7" print(f"[BUILD_NVEP] ensuring NCCL-EP floor --no-deps: {wheel}") @@ -732,6 +750,25 @@ def _install_nvep_runtime_wheels(built_nixl: bool) -> None: ) from e +def _compile_deps_installed(specs) -> bool: + """True when every spec is already satisfied. False on any doubt, so that an + unreadable environment still reaches the install below.""" + try: + from importlib.metadata import version + from packaging.requirements import Requirement + except ImportError: + return False + + for spec in specs: + try: + req = Requirement(spec) + if not req.specifier.contains(version(req.name), prereleases=True): + return False + except Exception: + return False + return True + + def _install_cuda_tile_compile_deps() -> None: """Install cuda-tile's compile chain with ``--no-deps`` to dodge libcudart.so.13. @@ -775,6 +812,18 @@ def _install_cuda_tile_compile_deps() -> None: "nvidia-nvjitlink<14,>=13.3", "nvidia-cuda-crt<13.4,>=13.2", ] + + if _compile_deps_installed(wheels): + print("[BUILD] cuda-tile compile deps already installed; skipping", flush=True) + return + + if _no_pip_installs(): + print( + "[BUILD] FLASHINFER_BUILD_NO_PIP=1; not installing cuda-tile compile deps", + flush=True, + ) + return + print(f"[BUILD] cuda-tile compile deps (--no-deps): {' '.join(wheels)}", flush=True) uv_bin = shutil.which("uv") diff --git a/ci/bash.sh b/ci/bash.sh index de110d13a27..a7dbbccfe2e 100644 --- a/ci/bash.sh +++ b/ci/bash.sh @@ -60,6 +60,27 @@ else COMMAND=("$@") fi +DOCKER_ENV="${DOCKER_ENV} -e PIP_RETRIES=10 -e PIP_DEFAULT_TIMEOUT=60" + +# Share pip's downloads with the other jobs that land on this reused runner. In +# HOME because the workspace is wiped between jobs and /opt needs root. +# CI_PIP_CACHE_DIR="" opts out. +: "${CI_PIP_CACHE_DIR=${HOME:-/tmp}/.cache/flashinfer-ci/pip}" +if [ -n "${CI_PIP_CACHE_DIR}" ] && mkdir -p "${CI_PIP_CACHE_DIR}" 2>/dev/null; then + CACHE_MB=$(du -sm "${CI_PIP_CACHE_DIR}" 2>/dev/null | cut -f1) + if [ "${CACHE_MB:-0}" -gt "${CI_PIP_CACHE_MAX_MB:-20480}" ]; then + echo "Clearing pip cache: ${CACHE_MB} MB exceeds ${CI_PIP_CACHE_MAX_MB:-20480} MB" + # Containers write into it as root, so the runner user may need sudo. + rm -rf "${CI_PIP_CACHE_DIR:?}" 2>/dev/null \ + || sudo -n rm -rf "${CI_PIP_CACHE_DIR:?}" 2>/dev/null || true + mkdir -p "${CI_PIP_CACHE_DIR}" 2>/dev/null || true + fi + DOCKER_VOLUMNS="${DOCKER_VOLUMNS} -v ${CI_PIP_CACHE_DIR}:/pip-cache" + DOCKER_ENV="${DOCKER_ENV} -e PIP_CACHE_DIR=/pip-cache" +else + echo "Shared pip cache disabled or not writable; downloads will not be reused" +fi + # Use nvidia-docker if the container is GPU. if [[ ${USE_GPU} == "true" ]]; then DOCKER_ENV="${DOCKER_ENV} -e CUDA_VISIBLE_DEVICES=${CUDA_VISIBLE_DEVICES}" diff --git a/docker/install/install_python_packages.sh b/docker/install/install_python_packages.sh index 4c4d2847eef..cd38b8901d6 100644 --- a/docker/install/install_python_packages.sh +++ b/docker/install/install_python_packages.sh @@ -53,7 +53,9 @@ fi pip3 install --upgrade "$CUDA_PYTHON" pip3 install -r /install/requirements.txt -pip3 install responses pytest scipy build "$NVSHMEM4PY" +# wheel: imported by flashinfer-jit-cache's build backend, which CI builds with +# --no-isolation. +pip3 install responses pytest scipy build wheel "$NVSHMEM4PY" pip3 install --upgrade "$CUDA_PYTHON" # Install cudnn package based on CUDA version diff --git a/scripts/check_requirements.py b/scripts/check_requirements.py new file mode 100755 index 00000000000..d5775848f6e --- /dev/null +++ b/scripts/check_requirements.py @@ -0,0 +1,71 @@ +#!/usr/bin/env python3 +"""Report which requirements the current environment does not already satisfy. + +Prints one unsatisfied requirement per line on stdout, ready to pass to +`pip install`. Exit codes: 0 all satisfied, 1 some printed, 2 the check could +not run, so callers can fall back to a full dependency sync. +""" + +import sys +from importlib.metadata import PackageNotFoundError, version + +try: + from packaging.requirements import InvalidRequirement, Requirement +except ImportError as exc: # pragma: no cover - packaging ships in every CI image + print(f"cannot check requirements: {exc}", file=sys.stderr) + raise SystemExit(2) from exc + + +def parse_requirements(path): + """Read a requirements file, ignoring comments, blanks, and pip options.""" + requirements = [] + try: + with open(path, encoding="utf-8") as handle: + lines = handle.read().splitlines() + except OSError as exc: + print(f"skipping {path}: {exc}", file=sys.stderr) + return requirements + + for raw in lines: + line = raw.split("#", 1)[0].strip() + if not line or line.startswith("-"): + continue + try: + requirements.append(Requirement(line)) + except InvalidRequirement as exc: + print(f"{path}: cannot parse {line!r}: {exc}", file=sys.stderr) + raise SystemExit(2) from exc + return requirements + + +def main(paths): + if not paths: + print("usage: check_requirements.py REQUIREMENTS_FILE...", file=sys.stderr) + return 2 + + unsatisfied = [] + for path in paths: + for req in parse_requirements(path): + if req.marker is not None and not req.marker.evaluate(): + continue + try: + installed = version(req.name) + except PackageNotFoundError: + print(f"{req.name}: not installed", file=sys.stderr) + unsatisfied.append(str(req)) + continue + if not req.specifier.contains(installed, prereleases=True): + print( + f"{req.name}: installed {installed} does not satisfy " + f"{req.specifier}", + file=sys.stderr, + ) + unsatisfied.append(str(req)) + + for req in unsatisfied: + print(req) + return 1 if unsatisfied else 0 + + +if __name__ == "__main__": + raise SystemExit(main(sys.argv[1:])) diff --git a/scripts/setup_test_env.sh b/scripts/setup_test_env.sh index ee5d5bc7e72..178c5e94049 100755 --- a/scripts/setup_test_env.sh +++ b/scripts/setup_test_env.sh @@ -31,6 +31,39 @@ if [ -z "${PIP_CONSTRAINT:-}" ]; then unset _torch_pin fi +# Install only what this branch moved past the image. Installing the full file +# instead lets one drifted floor re-resolve every other requirement, which is how +# torch has twice been swapped for a build that does not match the image. +_reqs_output=$(python "${SCRIPT_DIR}/check_requirements.py" \ + "${REPO_ROOT}/requirements.txt") && _reqs_status=0 || _reqs_status=$? +case "${_reqs_status}" in + 0) + echo "Requirements are satisfied by the image; nothing to install." + ;; + 1) + echo "Installing requirements this branch changed: ${_reqs_output//$'\n'/ }" + _reqs_list=() + while IFS= read -r _req; do + [ -n "${_req}" ] && _reqs_list+=("${_req}") + done <<< "${_reqs_output}" + pip install "${_reqs_list[@]}" + unset _reqs_list _req + ;; + *) + echo "WARNING: requirement check failed; syncing the full requirements" >&2 + pip install -r "${REPO_ROOT}/requirements.txt" + ;; +esac +unset _reqs_output _reqs_status + +# Install using only what the image carries; the check above covered the rest. +# FLASHINFER_BUILD_NO_PIP keeps dropping isolation from activating the build +# hooks' own downloads, which isolation has always swallowed here. +install_flashinfer_editable() { + FLASHINFER_BUILD_NO_PIP=1 \ + pip install --no-build-isolation --no-deps -e "${1:-.}" -v +} + # Source the environment override file if it exists if [ -f "${REPO_ROOT}/ci/setup_python.env" ]; then source "${REPO_ROOT}/ci/setup_python.env" diff --git a/scripts/task_jit_run_tests_part1.sh b/scripts/task_jit_run_tests_part1.sh index 5f82f868318..daeea17bbfc 100755 --- a/scripts/task_jit_run_tests_part1.sh +++ b/scripts/task_jit_run_tests_part1.sh @@ -10,7 +10,7 @@ set -x source "$(dirname "${BASH_SOURCE[0]}")/setup_test_env.sh" if [ "$SKIP_INSTALL" = "0" ]; then - pip install -e . -v + install_flashinfer_editable fi # Run each test file separately to isolate CUDA memory issues diff --git a/scripts/task_jit_run_tests_part2.sh b/scripts/task_jit_run_tests_part2.sh index 919810e31e3..29a5b5b367b 100755 --- a/scripts/task_jit_run_tests_part2.sh +++ b/scripts/task_jit_run_tests_part2.sh @@ -10,7 +10,7 @@ set -x source "$(dirname "${BASH_SOURCE[0]}")/setup_test_env.sh" if [ "$SKIP_INSTALL" = "0" ]; then - pip install -e . -v + install_flashinfer_editable fi # Run each test file separately to isolate CUDA memory issues diff --git a/scripts/task_jit_run_tests_part3.sh b/scripts/task_jit_run_tests_part3.sh index 03b70993298..208ff6b6ec1 100755 --- a/scripts/task_jit_run_tests_part3.sh +++ b/scripts/task_jit_run_tests_part3.sh @@ -10,7 +10,7 @@ set -x source "$(dirname "${BASH_SOURCE[0]}")/setup_test_env.sh" if [ "$SKIP_INSTALL" = "0" ]; then - pip install -e . -v + install_flashinfer_editable fi # Run each test file separately to isolate CUDA memory issues diff --git a/scripts/task_jit_run_tests_part4.sh b/scripts/task_jit_run_tests_part4.sh index 5407a4d3c79..0f5fc67ac50 100755 --- a/scripts/task_jit_run_tests_part4.sh +++ b/scripts/task_jit_run_tests_part4.sh @@ -10,7 +10,7 @@ set -x source "$(dirname "${BASH_SOURCE[0]}")/setup_test_env.sh" if [ "$SKIP_INSTALL" = "0" ]; then - pip install -e . -v + install_flashinfer_editable fi export PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True # avoid memory fragmentation diff --git a/scripts/task_jit_run_tests_part5.sh b/scripts/task_jit_run_tests_part5.sh index 2c38d20a22e..c3b17a88285 100755 --- a/scripts/task_jit_run_tests_part5.sh +++ b/scripts/task_jit_run_tests_part5.sh @@ -10,7 +10,7 @@ set -x source "$(dirname "${BASH_SOURCE[0]}")/setup_test_env.sh" if [ "$SKIP_INSTALL" = "0" ]; then - pip install -e . -v + install_flashinfer_editable fi # Run each test file separately to isolate CUDA memory issues diff --git a/scripts/task_test_jit_cache_package_build_import.sh b/scripts/task_test_jit_cache_package_build_import.sh index 3e27f645f87..26dae24db3a 100755 --- a/scripts/task_test_jit_cache_package_build_import.sh +++ b/scripts/task_test_jit_cache_package_build_import.sh @@ -193,7 +193,7 @@ if [[ "${CUDA_VERSION}" == 13* ]]; then FLASHINFER_EDITABLE_SPEC=".[cu13]" fi run_with_aot_memory_monitor "pip_install_flashinfer_editable" \ - pip install --no-build-isolation -e "${FLASHINFER_EDITABLE_SPEC}" -v || { + pip install --no-build-isolation --no-deps -e "${FLASHINFER_EDITABLE_SPEC}" -v || { echo "ERROR: Failed to install flashinfer package" exit 1 } @@ -216,8 +216,11 @@ echo "Building flashinfer-jit-cache wheel" echo "========================================" cd flashinfer-jit-cache rm -rf dist build *.egg-info +# The image satisfies this package's build requires except wheel, until the +# images carrying it are rolled out. +python -c "import wheel" 2>/dev/null || pip install --no-deps wheel run_with_aot_memory_monitor "build_flashinfer_jit_cache_wheel" \ - python -m build --wheel + python -m build --wheel --no-isolation # Get the built wheel file WHEEL_FILE=$(ls -t dist/*.whl | head -n 1) diff --git a/scripts/task_test_single_node_comm_kernels.sh b/scripts/task_test_single_node_comm_kernels.sh index 071a4e45cb4..12d9aa3d736 100644 --- a/scripts/task_test_single_node_comm_kernels.sh +++ b/scripts/task_test_single_node_comm_kernels.sh @@ -15,7 +15,7 @@ find . -type f -name '*.pyc' -delete 2>/dev/null || true echo "Cache cleaned." echo "" -pip install -e . -v +install_flashinfer_editable # nvshmem4py-cu12 pins cuda-python<=12.9; letting pip resolve its deps on a # cu13 container downgrades cuda-python/cuda-bindings and makes the next diff --git a/scripts/test_utils.sh b/scripts/test_utils.sh index 91d9e106180..8ecadbc16db 100755 --- a/scripts/test_utils.sh +++ b/scripts/test_utils.sh @@ -297,8 +297,10 @@ install_and_verify() { pip install --upgrade "nvidia-cutlass-dsl[cu13]==4.7.0" fi - # Install local python sources - pip install -e . -v --no-deps + # Install local python sources. The env var keeps --no-build-isolation + # from activating the build hooks' own downloads (see setup_test_env.sh). + FLASHINFER_BUILD_NO_PIP=1 \ + pip install -e . -v --no-deps --no-build-isolation echo "" # Verify installation From c074fe09322210e3d6ba2d34ef6f1fab7a75d539 Mon Sep 17 00:00:00 2001 From: Yong Wu Date: Tue, 18 Aug 2026 11:46:11 -0700 Subject: [PATCH 2/3] fix comments --- build_backend.py | 14 +++++++++++--- scripts/check_requirements.py | 4 ++-- .../task_test_jit_cache_package_build_import.sh | 1 + 3 files changed, 14 insertions(+), 5 deletions(-) diff --git a/build_backend.py b/build_backend.py index a58c6530160..c7d375a01f5 100644 --- a/build_backend.py +++ b/build_backend.py @@ -716,6 +716,13 @@ def _install_nvep_runtime_wheels(built_nixl: bool) -> None: wheels.append(f"nixl-cu{cuda_major}=={_NIXL_WHEEL_VERSION}") if not wheels: return + if _no_pip_installs(): + # Reachable only if the wheel was already present, since the pre-flight + # probe needs it to import before anything is built. + print( + "[BUILD_NVEP] FLASHINFER_BUILD_NO_PIP=1; runtime wheels left as installed" + ) + return print(f"[BUILD_NVEP] installing runtime wheels --no-deps: {' '.join(wheels)}") @@ -754,8 +761,9 @@ def _compile_deps_installed(specs) -> bool: """True when every spec is already satisfied. False on any doubt, so that an unreadable environment still reaches the install below.""" try: - from importlib.metadata import version - from packaging.requirements import Requirement + from importlib.metadata import PackageNotFoundError, version + from packaging.requirements import InvalidRequirement, Requirement + from packaging.version import InvalidVersion except ImportError: return False @@ -764,7 +772,7 @@ def _compile_deps_installed(specs) -> bool: req = Requirement(spec) if not req.specifier.contains(version(req.name), prereleases=True): return False - except Exception: + except (PackageNotFoundError, InvalidRequirement, InvalidVersion): return False return True diff --git a/scripts/check_requirements.py b/scripts/check_requirements.py index d5775848f6e..a521e6c086d 100755 --- a/scripts/check_requirements.py +++ b/scripts/check_requirements.py @@ -23,8 +23,8 @@ def parse_requirements(path): with open(path, encoding="utf-8") as handle: lines = handle.read().splitlines() except OSError as exc: - print(f"skipping {path}: {exc}", file=sys.stderr) - return requirements + print(f"cannot read {path}: {exc}", file=sys.stderr) + raise SystemExit(2) from exc for raw in lines: line = raw.split("#", 1)[0].strip() diff --git a/scripts/task_test_jit_cache_package_build_import.sh b/scripts/task_test_jit_cache_package_build_import.sh index 26dae24db3a..d609a11b035 100755 --- a/scripts/task_test_jit_cache_package_build_import.sh +++ b/scripts/task_test_jit_cache_package_build_import.sh @@ -193,6 +193,7 @@ if [[ "${CUDA_VERSION}" == 13* ]]; then FLASHINFER_EDITABLE_SPEC=".[cu13]" fi run_with_aot_memory_monitor "pip_install_flashinfer_editable" \ + env FLASHINFER_BUILD_NO_PIP=1 \ pip install --no-build-isolation --no-deps -e "${FLASHINFER_EDITABLE_SPEC}" -v || { echo "ERROR: Failed to install flashinfer package" exit 1 From e6438e899c8bb7d8428b99af4c1073a15d99394b Mon Sep 17 00:00:00 2001 From: Yong Wu Date: Tue, 18 Aug 2026 12:23:47 -0700 Subject: [PATCH 3/3] fix --- scripts/task_test_jit_cache_package_build_import.sh | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/scripts/task_test_jit_cache_package_build_import.sh b/scripts/task_test_jit_cache_package_build_import.sh index d609a11b035..7294580c561 100755 --- a/scripts/task_test_jit_cache_package_build_import.sh +++ b/scripts/task_test_jit_cache_package_build_import.sh @@ -220,8 +220,11 @@ rm -rf dist build *.egg-info # The image satisfies this package's build requires except wheel, until the # images carrying it are rolled out. python -c "import wheel" 2>/dev/null || pip install --no-deps wheel +# --no-isolation keeps build from re-downloading torch, but it then verifies the +# build requires transitively, and the image's torch declares an nvidia-cudnn +# wheel the image does not install. Testing what the image ships is the point. run_with_aot_memory_monitor "build_flashinfer_jit_cache_wheel" \ - python -m build --wheel --no-isolation + python -m build --wheel --no-isolation --skip-dependency-check # Get the built wheel file WHEEL_FILE=$(ls -t dist/*.whl | head -n 1)