From f5dc1e668129d60f3d4ec4407632d18d78583406 Mon Sep 17 00:00:00 2001 From: Jonathan Dierksen Date: Thu, 7 May 2026 23:04:22 +0000 Subject: [PATCH 1/8] build(jit-cache): split flashinfer-jit-cache wheels by SM family The cu130 flashinfer-jit-cache wheel grew to 2.0 GB and started failing to upload as a GitHub Release asset (per-asset 2 GiB limit; see #3257). Each new SM target appended ~150-200 MB compressed to every wheel, and cu130 carries 8 (sm75/80/89/90a/100a/103a/110a/120f). Split each (CUDA, CPU-arch) wheel into three by GPU SM family: - sm9x - Ampere/Ada/Hopper (<= sm90a) - sm10x - Datacenter Blackwell (sm100a/103a/110a) - sm12x - Consumer Blackwell (sm120f, future sm121a) Same package name everywhere; the family is encoded in the PEP 440 local-version, so wheels resolve as e.g. 'flashinfer-jit-cache== 0.6.11+cu130.sm10x'. Existing 'pip install flashinfer-jit-cache' still works once the right pin is given. Driven by a new 'flashinfer install-jit-cache-wheel' subcommand that detects FlashInfer version, CUDA version, and GPU compute capability (via torch.cuda.get_device_capability) and runs the matching pip install. Honors --cuda-version, --sm-family, --nightly, --dry-run. Modeled on the CLI scaffolding from #3142 with the family dimension added. Build side: 'FLASHINFER_JIT_CACHE_SM_FAMILY' env var, when set, filters 'FLASHINFER_CUDA_ARCH_LIST' to the family's archs and appends '.' to the local-version suffix. Release / nightly workflows gain an 'sm_family' matrix dimension; the upload-to-release loop iterates over all three families. The wheel-index regex accepts the new local-version shape and remains compatible with the legacy '+cuXY' format. Closes #3257 Co-Authored-By: Claude Opus 4.7 (1M context) --- .github/workflows/nightly-release.yml | 50 ++++--- .github/workflows/release.yml | 47 ++++--- README.md | 17 ++- docs/installation.rst | 34 ++++- flashinfer-jit-cache/build_backend.py | 78 +++++++++++ flashinfer/__main__.py | 188 ++++++++++++++++++++++++++ scripts/update_whl_index.py | 7 +- 7 files changed, 369 insertions(+), 52 deletions(-) diff --git a/.github/workflows/nightly-release.yml b/.github/workflows/nightly-release.yml index badae18923d..e8309976502 100644 --- a/.github/workflows/nightly-release.yml +++ b/.github/workflows/nightly-release.yml @@ -126,6 +126,7 @@ jobs: matrix: cuda: ["12.8", "12.9", "13.0"] arch: ['x86_64', 'aarch64'] + sm_family: ['sm9x', 'sm10x', 'sm12x'] runs-on: [self-hosted, linux, "${{ matrix.arch == 'aarch64' && 'arm64' || 'x64' }}", cpu, on-demand] @@ -144,6 +145,7 @@ jobs: echo "RAM: $(free -h | awk '/^Mem:/ {print $7 " available out of " $2}')" echo "Disk: $(df -h / | awk 'NR==2 {print $4 " available out of " $2}')" echo "Architecture: $(uname -m)" + echo "SM family: ${{ matrix.sm_family }}" - name: Checkout code uses: actions/checkout@v4 @@ -176,6 +178,7 @@ jobs: -e FLASHINFER_DEV_RELEASE_SUFFIX="${FLASHINFER_DEV_RELEASE_SUFFIX}" \ -e ARCH="${{ matrix.arch }}" \ -e FLASHINFER_CUDA_ARCH_LIST="${FLASHINFER_CUDA_ARCH_LIST}" \ + -e FLASHINFER_JIT_CACHE_SM_FAMILY="${{ matrix.sm_family }}" \ -w /workspace \ ${{ env.DOCKER_IMAGE }} \ bash /workspace/scripts/build_flashinfer_jit_cache_whl.sh @@ -188,7 +191,7 @@ jobs: id: artifact-name run: | CUDA_NO_DOT=$(echo "${{ matrix.cuda }}" | tr -d '.') - echo "name=jit-cache-cu${CUDA_NO_DOT}-${{ matrix.arch }}" >> $GITHUB_OUTPUT + echo "name=jit-cache-cu${CUDA_NO_DOT}-${{ matrix.arch }}-${{ matrix.sm_family }}" >> $GITHUB_OUTPUT - name: Upload flashinfer-jit-cache artifact uses: actions/upload-artifact@v4 @@ -252,29 +255,32 @@ jobs: env: GH_TOKEN: ${{ github.token }} run: | - # Upload jit-cache wheels one at a time to avoid OOM - # Each wheel can be several GB, so we download, upload, delete, repeat + # Upload jit-cache wheels one at a time to avoid OOM. + # Wheels are split per (CUDA, CPU-arch, SM family) to stay under + # GitHub Releases' 2 GiB per-asset limit. mkdir -p dist-jit-cache for cuda in 128 129 130; do for arch in x86_64 aarch64; do - ARTIFACT_NAME="jit-cache-cu${cuda}-${arch}" - echo "Processing ${ARTIFACT_NAME}..." - - # Download this specific artifact - gh run download ${{ github.run_id }} -n "${ARTIFACT_NAME}" -D dist-jit-cache/ || { - echo "Warning: Failed to download ${ARTIFACT_NAME}, skipping..." - continue - } - - # Upload to release - if [ -n "$(ls -A dist-jit-cache/)" ]; then - gh release upload "${{ needs.setup.outputs.release_tag }}" dist-jit-cache/* --clobber - echo "✅ Uploaded ${ARTIFACT_NAME}" - fi - - # Clean up to save disk space before next iteration - rm -rf dist-jit-cache/* + for sm_family in sm9x sm10x sm12x; do + ARTIFACT_NAME="jit-cache-cu${cuda}-${arch}-${sm_family}" + echo "Processing ${ARTIFACT_NAME}..." + + # Download this specific artifact + gh run download ${{ github.run_id }} -n "${ARTIFACT_NAME}" -D dist-jit-cache/ || { + echo "Warning: Failed to download ${ARTIFACT_NAME}, skipping..." + continue + } + + # Upload to release + if [ -n "$(ls -A dist-jit-cache/)" ]; then + gh release upload "${{ needs.setup.outputs.release_tag }}" dist-jit-cache/* --clobber + echo "✅ Uploaded ${ARTIFACT_NAME}" + fi + + # Clean up to save disk space before next iteration + rm -rf dist-jit-cache/* + done done done @@ -321,9 +327,11 @@ jobs: path: dist-cubin/ - name: Download flashinfer-jit-cache artifact + # The test runner is an sm86 (Ampere) host, which falls under the + # sm9x family wheel. Update the family if test runners change. uses: actions/download-artifact@v4 with: - name: jit-cache-cu${{ matrix.cuda == '12.9' && '129' || '130' }}-x86_64 + name: jit-cache-cu${{ matrix.cuda == '12.9' && '129' || '130' }}-x86_64-sm9x path: dist-jit-cache/ - name: Get Docker image tag diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index c25acf33933..80063ba923f 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -162,6 +162,7 @@ jobs: matrix: cuda: ["12.8", "12.9", "13.0"] arch: ['x86_64', 'aarch64'] + sm_family: ['sm9x', 'sm10x', 'sm12x'] runs-on: [self-hosted, linux, "${{ matrix.arch == 'aarch64' && 'arm64' || 'x64' }}", cpu, on-demand] @@ -172,6 +173,7 @@ jobs: echo "RAM: $(free -h | awk '/^Mem:/ {print $7 " available out of " $2}')" echo "Disk: $(df -h / | awk 'NR==2 {print $4 " available out of " $2}')" echo "Architecture: $(uname -m)" + echo "SM family: ${{ matrix.sm_family }}" - name: Checkout code uses: actions/checkout@v4 @@ -203,6 +205,7 @@ jobs: -e FLASHINFER_LOCAL_VERSION="$FLASHINFER_LOCAL_VERSION" \ -e ARCH="${{ matrix.arch }}" \ -e FLASHINFER_CUDA_ARCH_LIST="${FLASHINFER_CUDA_ARCH_LIST}" \ + -e FLASHINFER_JIT_CACHE_SM_FAMILY="${{ matrix.sm_family }}" \ -w /workspace \ ${{ env.DOCKER_IMAGE }} \ bash /workspace/scripts/build_flashinfer_jit_cache_whl.sh @@ -215,7 +218,7 @@ jobs: id: artifact-name run: | CUDA_NO_DOT=$(echo "${{ matrix.cuda }}" | tr -d '.') - echo "name=jit-cache-cu${CUDA_NO_DOT}-${{ matrix.arch }}" >> $GITHUB_OUTPUT + echo "name=jit-cache-cu${CUDA_NO_DOT}-${{ matrix.arch }}-${{ matrix.sm_family }}" >> $GITHUB_OUTPUT - name: Upload flashinfer-jit-cache artifact uses: actions/upload-artifact@v4 @@ -314,29 +317,33 @@ jobs: env: GH_TOKEN: ${{ github.token }} run: | - # Upload jit-cache wheels one at a time to avoid OOM - # Each wheel can be several GB, so we download, upload, delete, repeat + # Upload jit-cache wheels one at a time to avoid OOM. + # Each wheel can be ~1 GB, so we download, upload, delete, repeat. + # Wheels are split per (CUDA, CPU-arch, SM family) to stay under + # GitHub Releases' 2 GiB per-asset limit. mkdir -p dist-jit-cache for cuda in 128 129 130; do for arch in x86_64 aarch64; do - ARTIFACT_NAME="jit-cache-cu${cuda}-${arch}" - echo "Processing ${ARTIFACT_NAME}..." - - # Download this specific artifact - gh run download ${{ github.run_id }} -n "${ARTIFACT_NAME}" -D dist-jit-cache/ || { - echo "Warning: Failed to download ${ARTIFACT_NAME}, skipping..." - continue - } - - # Upload to release - if [ -n "$(ls -A dist-jit-cache/)" ]; then - gh release upload "${{ needs.setup.outputs.release_tag }}" dist-jit-cache/* --clobber - echo "✅ Uploaded ${ARTIFACT_NAME}" - fi - - # Clean up to save disk space before next iteration - rm -rf dist-jit-cache/* + for sm_family in sm9x sm10x sm12x; do + ARTIFACT_NAME="jit-cache-cu${cuda}-${arch}-${sm_family}" + echo "Processing ${ARTIFACT_NAME}..." + + # Download this specific artifact + gh run download ${{ github.run_id }} -n "${ARTIFACT_NAME}" -D dist-jit-cache/ || { + echo "Warning: Failed to download ${ARTIFACT_NAME}, skipping..." + continue + } + + # Upload to release + if [ -n "$(ls -A dist-jit-cache/)" ]; then + gh release upload "${{ needs.setup.outputs.release_tag }}" dist-jit-cache/* --clobber + echo "✅ Uploaded ${ARTIFACT_NAME}" + fi + + # Clean up to save disk space before next iteration + rm -rf dist-jit-cache/* + done done done diff --git a/README.md b/README.md index 142ec113cfd..68241f5ed89 100644 --- a/README.md +++ b/README.md @@ -102,10 +102,13 @@ pip install flashinfer-python ```bash pip install flashinfer-python flashinfer-cubin -# JIT cache (replace cu129 with your CUDA version) -pip install flashinfer-jit-cache --index-url https://flashinfer.ai/whl/cu129 +# JIT cache: autodetect CUDA + GPU SM family and run the matching pip install. +# Use --dry-run to preview, --sm-family / --cuda-version to override. +flashinfer install-jit-cache-wheel ``` +`flashinfer-jit-cache` is published as separate per-(CUDA, SM family) wheels — `sm9x` (Ampere/Ada/Hopper, ≤sm90), `sm10x` (Datacenter Blackwell, sm100/103/110), and `sm12x` (Consumer Blackwell, sm120/121) — because a single multi-arch wheel exceeds GitHub Releases' 2 GiB asset limit. The CLI resolves the right one for you. + **For Blackwell (SM100+) CuTe DSL kernels**, install with the CUDA 13 extra to enable Blackwell-optimized kernels: ```bash @@ -168,6 +171,12 @@ export FLASHINFER_CUDA_ARCH_LIST="7.5 8.0 8.9 9.0a 10.0a 10.3a 11.0a 12.0f" cd flashinfer-jit-cache python -m build --no-isolation --wheel python -m pip install dist/*.whl + +# Or build a per-SM-family wheel matching a release artifact (smaller; one GPU family). +# The local-version on the resulting wheel will include the family (e.g. +cu130.sm10x). +export FLASHINFER_CUDA_ARCH_LIST="10.0a 10.3a 11.0a" +export FLASHINFER_JIT_CACHE_SM_FAMILY="sm10x" +python -m build --no-isolation --wheel ``` For more details, see the [Install from Source documentation](https://docs.flashinfer.ai/installation.html#install-from-source). @@ -178,8 +187,8 @@ For more details, see the [Install from Source documentation](https://docs.flash pip install -U --pre flashinfer-python --index-url https://flashinfer.ai/whl/nightly/ --no-deps pip install flashinfer-python # Install dependencies from PyPI pip install -U --pre flashinfer-cubin --index-url https://flashinfer.ai/whl/nightly/ -# JIT cache (replace cu129 with your CUDA version) -pip install -U --pre flashinfer-jit-cache --index-url https://flashinfer.ai/whl/nightly/cu129 +# JIT cache: autodetect CUDA + GPU and pull from the nightly index +flashinfer install-jit-cache-wheel --nightly ``` ### CLI Tools diff --git a/docs/installation.rst b/docs/installation.rst index 14e36f75c82..df52d48973a 100644 --- a/docs/installation.rst +++ b/docs/installation.rst @@ -43,11 +43,26 @@ FlashInfer provides three packages: .. code-block:: bash pip install flashinfer-python flashinfer-cubin - # JIT cache package (replace cu129 with your CUDA version: cu128, cu129, or cu130) - pip install flashinfer-jit-cache --index-url https://flashinfer.ai/whl/cu129 + # JIT cache package: autodetects CUDA + GPU SM family and runs the right pip install. + flashinfer install-jit-cache-wheel This eliminates compilation and downloading overhead at runtime. +``flashinfer-jit-cache`` is published as separate per-(CUDA, SM family) wheels because a +single multi-arch wheel exceeds GitHub Releases' 2 GiB asset limit. The CLI resolves +the right one for you. To pick manually, override the autodetection: + +.. code-block:: bash + + # Datacenter Blackwell (sm100/103/110), CUDA 13.0 + flashinfer install-jit-cache-wheel --cuda-version 13.0 --sm-family sm10x + + # Show the pip command without executing + flashinfer install-jit-cache-wheel --dry-run + +The SM families are: ``sm9x`` (Ampere/Ada/Hopper, ≤sm90), ``sm10x`` (Datacenter Blackwell, +sm100/103/110), and ``sm12x`` (Consumer Blackwell, sm120/121). + .. _install-from-source: @@ -112,6 +127,17 @@ You can follow the steps below to install FlashInfer from source code: python -m build --no-isolation --wheel python -m pip install dist/*.whl + To build a per-SM-family wheel matching a release artifact (smaller; targets one + GPU family), set ``FLASHINFER_JIT_CACHE_SM_FAMILY``. The local-version suffix on + the resulting wheel will include the family (e.g. ``+cu130.sm10x``). + + .. code-block:: bash + + export FLASHINFER_CUDA_ARCH_LIST="10.0a 10.3a 11.0a" + export FLASHINFER_JIT_CACHE_SM_FAMILY="sm10x" + cd flashinfer-jit-cache + python -m build --no-isolation --wheel + Install Nightly Build ^^^^^^^^^^^^^^^^^^^^^^ @@ -124,8 +150,8 @@ Nightly builds are available for testing the latest features: pip install -U --pre flashinfer-python --index-url https://flashinfer.ai/whl/nightly/ --no-deps # Install the nightly package from custom index, without installing dependencies pip install flashinfer-python # Install flashinfer-python's dependencies from PyPI pip install -U --pre flashinfer-cubin --index-url https://flashinfer.ai/whl/nightly/ - # JIT cache package (replace cu129 with your CUDA version: cu128, cu129, or cu130) - pip install -U --pre flashinfer-jit-cache --index-url https://flashinfer.ai/whl/nightly/cu129 + # JIT cache package: autodetect CUDA + GPU and pull from the nightly index + flashinfer install-jit-cache-wheel --nightly Verify Installation ^^^^^^^^^^^^^^^^^^^ diff --git a/flashinfer-jit-cache/build_backend.py b/flashinfer-jit-cache/build_backend.py index 1a13fd7e2ad..b3c634aa060 100644 --- a/flashinfer-jit-cache/build_backend.py +++ b/flashinfer-jit-cache/build_backend.py @@ -30,6 +30,71 @@ os.environ["FLASHINFER_DISABLE_VERSION_CHECK"] = "1" +# SM family → arch list filter. Each entry is the predicate `(major, minor_str) -> bool` +# applied to entries in FLASHINFER_CUDA_ARCH_LIST. The local-version suffix encodes the +# family so users (and pip) can resolve the right wheel: e.g. "0.6.11+cu130.sm10x". +# +# Keep this in sync with `_detect_sm_family` in flashinfer/__main__.py. +SM_FAMILIES = { + "sm9x": lambda major, minor: major < 10, # 7.5 / 8.0 / 8.9 / 9.0a (Ampere/Ada/Hopper) + "sm10x": lambda major, minor: 10 <= major < 12, # 10.0a / 10.3a / 11.0a (Datacenter Blackwell) + "sm12x": lambda major, minor: major >= 12, # 12.0f / 12.1a (Consumer Blackwell) +} + + +def _filter_arch_list_for_family(arch_list: str, family: str) -> str: + """Filter a space-separated FLASHINFER_CUDA_ARCH_LIST to only entries belonging to `family`.""" + predicate = SM_FAMILIES[family] + kept = [] + for entry in arch_list.split(): + major_str, minor_str = entry.split(".", 1) + major = int(major_str) + # `minor_str` may carry a suffix like 'a' or 'f' — keep the whole string for output, + # but parse leading digits for the comparison. + leading_digits = "".join(c for c in minor_str if c.isdigit()) + minor = int(leading_digits) if leading_digits else 0 + if predicate(major, minor): + kept.append(entry) + return " ".join(kept) + + +def _resolve_sm_family() -> str: + """Return the SM family this build targets, or '' for a legacy multi-family build.""" + family = os.environ.get("FLASHINFER_JIT_CACHE_SM_FAMILY", "").strip().lower() + if not family: + return "" + if family not in SM_FAMILIES: + raise RuntimeError( + f"Invalid FLASHINFER_JIT_CACHE_SM_FAMILY={family!r}. " + f"Expected one of: {sorted(SM_FAMILIES)}" + ) + return family + + +def _apply_sm_family_filter() -> str: + """If a family is selected, narrow FLASHINFER_CUDA_ARCH_LIST in-place and return the family suffix.""" + family = _resolve_sm_family() + if not family: + return "" + + arch_list = os.environ.get("FLASHINFER_CUDA_ARCH_LIST") + if not arch_list: + # The downstream build will fail with a clear error in compile_and_package_modules; + # we let it raise there to keep error messages consistent. + return family + + filtered = _filter_arch_list_for_family(arch_list, family) + if not filtered: + raise RuntimeError( + f"FLASHINFER_JIT_CACHE_SM_FAMILY={family} but FLASHINFER_CUDA_ARCH_LIST=" + f"{arch_list!r} contains no archs in that family. " + f"Set FLASHINFER_CUDA_ARCH_LIST to include archs matching {family}." + ) + print(f"SM family {family}: filtering FLASHINFER_CUDA_ARCH_LIST {arch_list!r} -> {filtered!r}") + os.environ["FLASHINFER_CUDA_ARCH_LIST"] = filtered + return family + + def _create_build_metadata(): """Create build metadata file with version information.""" version_file = Path(__file__).parent.parent / "version.txt" @@ -49,6 +114,13 @@ def _create_build_metadata(): # Append local version suffix if available local_version = os.environ.get("FLASHINFER_LOCAL_VERSION") + + # When this build targets a single SM family, append it to the local-version + # so users can pin e.g. "flashinfer-jit-cache==0.6.11+cu130.sm10x". + family = _resolve_sm_family() + if family: + local_version = f"{local_version}.{family}" if local_version else family + if local_version: # Use + to create a local version identifier that will appear in wheel name version = f"{version}+{local_version}" @@ -157,6 +229,12 @@ def _build_aot_modules(): def _prepare_build(): """Shared preparation logic for both wheel and editable builds.""" + _apply_sm_family_filter() + # Re-derive the build metadata so the family suffix is reflected in + # `_build_meta.py` even when the import-time call ran before the env var + # was set (e.g. when callers configure FLASHINFER_JIT_CACHE_SM_FAMILY in + # the same process before invoking build_wheel). + _create_build_metadata() _build_aot_modules() diff --git a/flashinfer/__main__.py b/flashinfer/__main__.py index eed66713597..54d382ace04 100644 --- a/flashinfer/__main__.py +++ b/flashinfer/__main__.py @@ -16,7 +16,11 @@ # flashinfer-cli import os +import subprocess +import sys + import click +from packaging.version import InvalidVersion, Version from tabulate import tabulate # type: ignore[import-untyped] from .artifacts import ( @@ -60,6 +64,44 @@ def _ensure_modules_registered(): return statuses +# SM family detection. Each family corresponds to a separately-built +# `flashinfer-jit-cache` wheel; the family is encoded in the local-version +# suffix (e.g. "0.6.11+cu130.sm10x"). Keep in sync with `SM_FAMILIES` in +# `flashinfer-jit-cache/build_backend.py`. +_SM_FAMILY_ORDER = ("sm9x", "sm10x", "sm12x") + + +def _sm_family_for_capability(major: int, minor: int) -> str: + if major < 10: + return "sm9x" # Ampere/Ada/Hopper + if major < 12: + return "sm10x" # Datacenter Blackwell (sm100/sm103/sm110) + return "sm12x" # Consumer Blackwell (sm120/sm121) + + +def _detect_sm_family() -> str: + """Return the SM family of the first visible CUDA device, or raise ClickException. + + Used as the default for `flashinfer install-jit-cache-wheel`. + """ + try: + import torch + except ImportError as e: # pragma: no cover - torch is a hard dep but be defensive + raise click.ClickException( + "PyTorch is required to autodetect the GPU architecture. " + "Install torch or pass --sm-family explicitly." + ) from e + + if not torch.cuda.is_available() or torch.cuda.device_count() == 0: + raise click.ClickException( + "No CUDA device visible — cannot autodetect SM family. " + "Pass --sm-family {sm9x|sm10x|sm12x} or run on a host with the target GPU." + ) + + major, minor = torch.cuda.get_device_capability(0) + return _sm_family_for_capability(major, minor) + + @click.group(invoke_without_command=True) @click.option( "--download-cubin", "download_cubin_flag", is_flag=True, help="Download artifacts" @@ -216,6 +258,152 @@ def clear_cubin_cmd(): click.secho(f"❌ Cubin clear failed: {e}", fg="red") +def _parse_cuda_version(cuda_version): + """Accepts None / "12.9" / "cu129"; returns a packaging Version.""" + if cuda_version is None: + return get_cuda_version() + + normalized = cuda_version.strip().lower() + if normalized.startswith("cu"): + digits = normalized[2:] + if not digits.isdigit() or len(digits) < 3: + raise click.ClickException( + "CUDA version must look like '12.9' or 'cu129'." + ) + normalized = f"{int(digits[:2])}.{int(digits[2:])}" + try: + return Version(normalized) + except InvalidVersion as e: + raise click.ClickException( + f"Invalid CUDA version '{cuda_version}'. Use formats like '12.9' or 'cu129'." + ) from e + + +def _cuda_index_label(cuda_version: Version) -> str: + return f"cu{cuda_version.major}{cuda_version.minor}" + + +def _build_jit_cache_requirement( + flashinfer_version: str, cuda_index_label: str, sm_family: str +) -> str: + if flashinfer_version == "0.0.0+unknown": + raise click.ClickException( + "Could not determine the installed FlashInfer version. " + "Pass --flashinfer-version explicitly." + ) + try: + parsed = Version(flashinfer_version) + except InvalidVersion as e: + raise click.ClickException( + f"Invalid FlashInfer version '{flashinfer_version}'." + ) from e + if parsed.local is not None: + raise click.ClickException( + "FlashInfer versions with a local-version suffix are not supported " + "for automatic flashinfer-jit-cache resolution. Pass --flashinfer-version " + "with a public version such as '0.4.1'." + ) + return f"flashinfer-jit-cache=={parsed.public}+{cuda_index_label}.{sm_family}" + + +def _build_jit_cache_index_url(cuda_index_label: str, nightly: bool) -> str: + if nightly: + return f"https://flashinfer.ai/whl/nightly/{cuda_index_label}" + return f"https://flashinfer.ai/whl/{cuda_index_label}" + + +@cli.command("install-jit-cache-wheel") +@click.option( + "--cuda-version", + default=None, + help="Override CUDA version detection, e.g. '12.9' or 'cu129'.", +) +@click.option( + "--sm-family", + default=None, + type=click.Choice(_SM_FAMILY_ORDER), + help=( + "Override SM family detection. sm9x = Ampere/Ada/Hopper (≤sm90), " + "sm10x = Datacenter Blackwell (sm100/103/110), sm12x = Consumer Blackwell (sm120/121)." + ), +) +@click.option( + "--flashinfer-version", + default=None, + help="Override FlashInfer version (defaults to the installed one).", +) +@click.option( + "--index-url", + default=None, + help="Explicit wheel index URL (overrides the autogenerated FlashInfer URL).", +) +@click.option( + "--nightly", + is_flag=True, + help="Install from the nightly wheel index instead of the release index.", +) +@click.option( + "--dry-run", + is_flag=True, + help="Print the pip command without executing it.", +) +def install_jit_cache_wheel_cmd( + cuda_version, sm_family, flashinfer_version, index_url, nightly, dry_run +): + """Install the matching flashinfer-jit-cache wheel for this GPU + CUDA. + + Detects FlashInfer version, CUDA version, and GPU compute capability, then runs + `pip install flashinfer-jit-cache==+cu.` against the FlashInfer + wheel index. Use --sm-family to bypass GPU detection (e.g. when prepping a + container image on a CPU-only host). + """ + resolved_cuda = _parse_cuda_version(cuda_version) + cuda_label = _cuda_index_label(resolved_cuda) + resolved_family = sm_family or _detect_sm_family() + resolved_flashinfer_version = flashinfer_version or __version__ + requirement = _build_jit_cache_requirement( + resolved_flashinfer_version, cuda_label, resolved_family + ) + resolved_index_url = index_url or _build_jit_cache_index_url(cuda_label, nightly) + + cmd = [ + sys.executable, + "-m", + "pip", + "install", + "--upgrade", + "--no-deps", + ] + if nightly: + cmd.append("--pre") + cmd.extend(["--index-url", resolved_index_url, requirement]) + + click.secho("=== JIT Cache Wheel Install ===", fg="yellow") + click.secho("FlashInfer version:", fg="magenta", nl=False) + click.secho(f" {resolved_flashinfer_version}", fg="cyan") + click.secho("CUDA version:", fg="magenta", nl=False) + click.secho(f" {resolved_cuda}", fg="cyan") + click.secho("SM family:", fg="magenta", nl=False) + click.secho(f" {resolved_family}", fg="cyan") + click.secho("Wheel index:", fg="magenta", nl=False) + click.secho(f" {resolved_index_url}", fg="cyan") + click.secho("Requirement:", fg="magenta", nl=False) + click.secho(f" {requirement}", fg="cyan") + click.secho("Command:", fg="magenta", nl=False) + click.secho(f" {' '.join(cmd)}", fg="cyan") + + if dry_run: + click.secho("Dry run requested; pip install was not executed.", fg="yellow") + return + + result = subprocess.run(cmd, check=False) + if result.returncode != 0: + raise click.ClickException( + f"pip install failed with exit code {result.returncode}." + ) + click.secho("✅ flashinfer-jit-cache installed successfully.", fg="green") + + @cli.command("module-status") @click.option("--detailed", is_flag=True, help="Show detailed information") @click.option( diff --git a/scripts/update_whl_index.py b/scripts/update_whl_index.py index cb9ed3d1838..ff194f27647 100644 --- a/scripts/update_whl_index.py +++ b/scripts/update_whl_index.py @@ -58,10 +58,11 @@ def get_package_info(wheel_path: pathlib.Path) -> Optional[dict]: "cuda": None, } - # Try flashinfer-jit-cache pattern (has CUDA suffix in version) - # Supports PEP 440: base_version[{a|b|rc}N][.postN][.devN]+cuXXX + # Try flashinfer-jit-cache pattern (has CUDA suffix in version, optionally + # followed by an SM-family qualifier such as ".sm10x"). + # Supports PEP 440: base_version[{a|b|rc}N][.postN][.devN]+cuXXX[.smYYY] match = re.match( - r"flashinfer_jit_cache-([0-9.]+(?:(?:a|b|rc)\d+)?(?:\.post\d+)?(?:\.dev\d+)?\+cu\d+)-", + r"flashinfer_jit_cache-([0-9.]+(?:(?:a|b|rc)\d+)?(?:\.post\d+)?(?:\.dev\d+)?\+cu\d+(?:\.sm\w+)?)-", wheel_name, ) if match: From c90d5b7500630a70fdebaff16410535807147090 Mon Sep 17 00:00:00 2001 From: Jonathan Dierksen Date: Thu, 7 May 2026 23:31:03 +0000 Subject: [PATCH 2/8] ci(jit-cache): add sm_family dimension to AOT Build Import matrix MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Mirrors the per-family split in release/nightly so PR CI actually exercises the per-family build path. Was previously running one job per (cuda, arch) which still built every arch; now runs three jobs per (cuda, arch) — one per SM family — each compiling only its family's archs. - pr-test.yml: 'aot-build-import' and 'aot-build-import-rerun' gain 'sm_family: [sm9x, sm10x, sm12x]'. cu126 is excluded for sm10x and sm12x because that toolkit only supports archs <= sm90. The rerun matrix builder mirrors the same exclude. FLASHINFER_JIT_CACHE_SM_FAMILY is forwarded into the test container via ci/bash.sh's '-e' flag. - task_test_jit_cache_package_build_import.sh: when FLASHINFER_JIT_CACHE_SM_FAMILY is set, filter FLASHINFER_CUDA_ARCH_LIST to that family's archs before running the wheel build and verify_all_modules_compiled.py. The build-side filter in build_backend.py mutates os.environ inside its own process only, so doing it once in the parent shell ensures both subprocesses see the same arch list. Also fix black formatting flagged by pre-commit on PR #3265: - build_backend.py: rewrite SM_FAMILIES lambdas as named functions to avoid black's awkward multi-line break of '<' chained comparisons. - __main__.py: collapse a ClickException to single-line per black's preference. Co-Authored-By: Claude Opus 4.7 (1M context) --- .github/workflows/pr-test.yml | 30 ++++++++++--- flashinfer-jit-cache/build_backend.py | 31 +++++++++---- flashinfer/__main__.py | 4 +- ...ask_test_jit_cache_package_build_import.sh | 45 +++++++++++++++++++ 4 files changed, 94 insertions(+), 16 deletions(-) diff --git a/.github/workflows/pr-test.yml b/.github/workflows/pr-test.yml index 85cf7ee027d..3840240d820 100644 --- a/.github/workflows/pr-test.yml +++ b/.github/workflows/pr-test.yml @@ -184,7 +184,7 @@ jobs: # AOT Build Import Tests (Spot + On-Demand Rerun) # --------------------------------------------------------------------------- aot-build-import: - name: AOT Build Import (${{ matrix.arch }}, ${{ matrix.cuda }}) + name: AOT Build Import (${{ matrix.arch }}, ${{ matrix.cuda }}, ${{ matrix.sm_family }}) needs: [gate, setup] if: | needs.gate.outputs.authorized == 'true' && @@ -202,6 +202,14 @@ jobs: matrix: arch: [x64, arm64] cuda: [cu126, cu128, cu129, cu130] + sm_family: [sm9x, sm10x, sm12x] + # cu126 only has sm9x archs (7.5 / 8.0 / 8.9 / 9.0a), so the + # sm10x and sm12x family builds would have empty arch lists. + exclude: + - cuda: cu126 + sm_family: sm10x + - cuda: cu126 + sm_family: sm12x env: DOCKER_IMAGE: flashinfer/flashinfer-ci-${{ matrix.cuda }}:${{ needs.setup.outputs.docker_tag }} steps: @@ -233,7 +241,10 @@ jobs: BUILD_NUMBER: ${{ github.run_number }} - name: Run Test - run: bash ci/bash.sh ${DOCKER_IMAGE} --no-gpu ./scripts/task_test_jit_cache_package_build_import.sh + run: | + bash ci/bash.sh ${DOCKER_IMAGE} --no-gpu \ + -e FLASHINFER_JIT_CACHE_SM_FAMILY ${{ matrix.sm_family }} \ + ./scripts/task_test_jit_cache_package_build_import.sh analyze-aot-failure: name: Analyze AOT Failure @@ -263,14 +274,20 @@ jobs: MATRIX='{"include":[' for arch in x64 arm64; do for cuda in cu126 cu128 cu129 cu130; do - MATRIX+='{"arch":"'$arch'","cuda":"'$cuda'"},' + for sm_family in sm9x sm10x sm12x; do + # Mirror the cu126 exclude rule from `aot-build-import`. + if [ "$cuda" = "cu126" ] && [ "$sm_family" != "sm9x" ]; then + continue + fi + MATRIX+='{"arch":"'$arch'","cuda":"'$cuda'","sm_family":"'$sm_family'"},' + done done done MATRIX="${MATRIX%,}]}" echo "rerun_matrix=$MATRIX" >> $GITHUB_OUTPUT aot-build-import-rerun: - name: AOT Build Import Rerun (${{ matrix.arch }}, ${{ matrix.cuda }}) + name: AOT Build Import Rerun (${{ matrix.arch }}, ${{ matrix.cuda }}, ${{ matrix.sm_family }}) needs: [setup, analyze-aot-failure] if: | !cancelled() && @@ -314,7 +331,10 @@ jobs: BUILD_NUMBER: ${{ github.run_number }} - name: Run Test - run: bash ci/bash.sh ${DOCKER_IMAGE} --no-gpu ./scripts/task_test_jit_cache_package_build_import.sh + run: | + bash ci/bash.sh ${DOCKER_IMAGE} --no-gpu \ + -e FLASHINFER_JIT_CACHE_SM_FAMILY ${{ matrix.sm_family }} \ + ./scripts/task_test_jit_cache_package_build_import.sh # --------------------------------------------------------------------------- # GPU JIT Tests - SM86 (A10G) - Spot + On-Demand Rerun diff --git a/flashinfer-jit-cache/build_backend.py b/flashinfer-jit-cache/build_backend.py index b3c634aa060..f79779d5c15 100644 --- a/flashinfer-jit-cache/build_backend.py +++ b/flashinfer-jit-cache/build_backend.py @@ -30,15 +30,27 @@ os.environ["FLASHINFER_DISABLE_VERSION_CHECK"] = "1" -# SM family → arch list filter. Each entry is the predicate `(major, minor_str) -> bool` -# applied to entries in FLASHINFER_CUDA_ARCH_LIST. The local-version suffix encodes the -# family so users (and pip) can resolve the right wheel: e.g. "0.6.11+cu130.sm10x". +# SM family → arch-list filter predicate `(major, minor_int) -> bool`, applied to entries +# in FLASHINFER_CUDA_ARCH_LIST. The local-version suffix encodes the family so users (and +# pip) can resolve the right wheel: e.g. "0.6.11+cu130.sm10x". # -# Keep this in sync with `_detect_sm_family` in flashinfer/__main__.py. +# Keep in sync with `_sm_family_for_capability` in flashinfer/__main__.py. +def _is_sm9x(major, minor): # 7.5 / 8.0 / 8.9 / 9.0a — Ampere/Ada/Hopper + return major < 10 + + +def _is_sm10x(major, minor): # 10.0a / 10.3a / 11.0a — Datacenter Blackwell + return 10 <= major < 12 + + +def _is_sm12x(major, minor): # 12.0f / 12.1a — Consumer Blackwell + return major >= 12 + + SM_FAMILIES = { - "sm9x": lambda major, minor: major < 10, # 7.5 / 8.0 / 8.9 / 9.0a (Ampere/Ada/Hopper) - "sm10x": lambda major, minor: 10 <= major < 12, # 10.0a / 10.3a / 11.0a (Datacenter Blackwell) - "sm12x": lambda major, minor: major >= 12, # 12.0f / 12.1a (Consumer Blackwell) + "sm9x": _is_sm9x, + "sm10x": _is_sm10x, + "sm12x": _is_sm12x, } @@ -90,7 +102,10 @@ def _apply_sm_family_filter() -> str: f"{arch_list!r} contains no archs in that family. " f"Set FLASHINFER_CUDA_ARCH_LIST to include archs matching {family}." ) - print(f"SM family {family}: filtering FLASHINFER_CUDA_ARCH_LIST {arch_list!r} -> {filtered!r}") + print( + f"SM family {family}: filtering FLASHINFER_CUDA_ARCH_LIST " + f"{arch_list!r} -> {filtered!r}" + ) os.environ["FLASHINFER_CUDA_ARCH_LIST"] = filtered return family diff --git a/flashinfer/__main__.py b/flashinfer/__main__.py index 54d382ace04..0f16c509f68 100644 --- a/flashinfer/__main__.py +++ b/flashinfer/__main__.py @@ -267,9 +267,7 @@ def _parse_cuda_version(cuda_version): if normalized.startswith("cu"): digits = normalized[2:] if not digits.isdigit() or len(digits) < 3: - raise click.ClickException( - "CUDA version must look like '12.9' or 'cu129'." - ) + raise click.ClickException("CUDA version must look like '12.9' or 'cu129'.") normalized = f"{int(digits[:2])}.{int(digits[2:])}" try: return Version(normalized) diff --git a/scripts/task_test_jit_cache_package_build_import.sh b/scripts/task_test_jit_cache_package_build_import.sh index 7143dbe6c4d..1904aa95590 100755 --- a/scripts/task_test_jit_cache_package_build_import.sh +++ b/scripts/task_test_jit_cache_package_build_import.sh @@ -64,6 +64,51 @@ print(" ".join(arches)) ') echo "FLASHINFER_CUDA_ARCH_LIST: ${FLASHINFER_CUDA_ARCH_LIST}" +# If a single SM family is requested, narrow the arch list so both the wheel +# build and the post-install verification (verify_all_modules_compiled.py) +# see the same set of archs. The build-side filter in build_backend.py mirrors +# this; we apply it here too so the verifier subprocess inherits the right +# environment (build_backend mutates os.environ inside its own process only). +# Keep in sync with SM_FAMILIES in flashinfer-jit-cache/build_backend.py. +if [ -n "${FLASHINFER_JIT_CACHE_SM_FAMILY}" ]; then + echo "Filtering arch list for FLASHINFER_JIT_CACHE_SM_FAMILY=${FLASHINFER_JIT_CACHE_SM_FAMILY}" + FAMILY="${FLASHINFER_JIT_CACHE_SM_FAMILY}" + case "${FAMILY}" in + sm9x|sm10x|sm12x) ;; + *) echo "ERROR: invalid FLASHINFER_JIT_CACHE_SM_FAMILY: ${FAMILY}"; exit 1;; + esac + FILTERED="" + for entry in ${FLASHINFER_CUDA_ARCH_LIST}; do + major="${entry%%.*}" + keep=false + case "${FAMILY}" in + sm9x) + if [ "${major}" -lt 10 ]; then keep=true; fi + ;; + sm10x) + if [ "${major}" -ge 10 ] && [ "${major}" -lt 12 ]; then keep=true; fi + ;; + sm12x) + if [ "${major}" -ge 12 ]; then keep=true; fi + ;; + esac + if [ "${keep}" = true ]; then + if [ -z "${FILTERED}" ]; then + FILTERED="${entry}" + else + FILTERED="${FILTERED} ${entry}" + fi + fi + done + if [ -z "${FILTERED}" ]; then + echo "ERROR: family ${FAMILY} has no matching archs in '${FLASHINFER_CUDA_ARCH_LIST}'." + echo " This (CUDA, family) combination should be excluded in the workflow matrix." + exit 1 + fi + export FLASHINFER_CUDA_ARCH_LIST="${FILTERED}" + echo "FLASHINFER_CUDA_ARCH_LIST (filtered): ${FLASHINFER_CUDA_ARCH_LIST}" +fi + echo "" echo "Current PyTorch version:" python -c "import torch; print(torch.__version__)" From 67adb489d618669e82752ad75e084c2ccff299e8 Mon Sep 17 00:00:00 2001 From: Jonathan Dierksen Date: Thu, 7 May 2026 23:54:18 +0000 Subject: [PATCH 3/8] ci(release): fix Release workflow checkout for fork PRs The Release workflow has a 'pull_request: paths: .github/workflows/ release.yml' trigger that runs the build jobs in dry-run mode whenever release.yml changes. Its checkout used: ref: ${{ github.event_name == 'pull_request' && github.head_ref || inputs.tag }} For a fork PR, github.head_ref resolves to a branch that doesn't exist on flashinfer-ai/flashinfer (because actions/checkout defaults 'repository:' to the workflow's repo). 'git fetch' fails three times, and the setup job dies before any actual build work runs. The bug has been latent since #1910 (2025-10-10), where the trigger and the buggy checkout were introduced together. It only fires on fork-PRs that touch release.yml; PRs from branches on the main repo work fine because the default 'repository:' already matches. Fix all four affected checkouts to set 'repository:' explicitly to the PR head's repo and pin to head.sha (which is also stable across re-pushes during the run). On workflow_dispatch the existing 'inputs.tag' path is unchanged. Co-Authored-By: Claude Opus 4.7 (1M context) --- .github/workflows/release.yml | 28 ++++++++++++++++++++++++---- 1 file changed, 24 insertions(+), 4 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 80063ba923f..f35ff654028 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -42,7 +42,12 @@ jobs: - name: Checkout code uses: actions/checkout@v4 with: - ref: ${{ github.event_name == 'pull_request' && github.head_ref || inputs.tag }} + # On PRs the head branch may live on a fork, so we must explicitly + # set `repository:` to the head's repo and pin to head.sha (stable + # across re-pushes during the run). On `workflow_dispatch` the + # default repository is correct and we check out the input tag. + ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || inputs.tag }} + repository: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name || github.repository }} submodules: true - name: Verify tag matches version.txt @@ -90,7 +95,12 @@ jobs: - name: Checkout code uses: actions/checkout@v4 with: - ref: ${{ github.event_name == 'pull_request' && github.head_ref || inputs.tag }} + # On PRs the head branch may live on a fork, so we must explicitly + # set `repository:` to the head's repo and pin to head.sha (stable + # across re-pushes during the run). On `workflow_dispatch` the + # default repository is correct and we check out the input tag. + ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || inputs.tag }} + repository: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name || github.repository }} submodules: true - name: Set up Python @@ -124,7 +134,12 @@ jobs: - name: Checkout code uses: actions/checkout@v4 with: - ref: ${{ github.event_name == 'pull_request' && github.head_ref || inputs.tag }} + # On PRs the head branch may live on a fork, so we must explicitly + # set `repository:` to the head's repo and pin to head.sha (stable + # across re-pushes during the run). On `workflow_dispatch` the + # default repository is correct and we check out the input tag. + ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || inputs.tag }} + repository: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name || github.repository }} submodules: true - name: Set up Python @@ -178,7 +193,12 @@ jobs: - name: Checkout code uses: actions/checkout@v4 with: - ref: ${{ github.event_name == 'pull_request' && github.head_ref || inputs.tag }} + # On PRs the head branch may live on a fork, so we must explicitly + # set `repository:` to the head's repo and pin to head.sha (stable + # across re-pushes during the run). On `workflow_dispatch` the + # default repository is correct and we check out the input tag. + ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || inputs.tag }} + repository: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name || github.repository }} submodules: true - name: Build wheel in container From 60974ee8a1f5e9df51f9cb13dc7c842c64816d0d Mon Sep 17 00:00:00 2001 From: dierksen Date: Fri, 8 May 2026 14:55:40 -0500 Subject: [PATCH 4/8] Address jit-cache review feedback --- build_utils.py | 44 +++++++++++++++++ flashinfer-jit-cache/build_backend.py | 52 +++----------------- flashinfer/__main__.py | 36 +++++++------- tests/cli/test_cli_cmds.py | 70 +++++++++++++++++++++++++++ 4 files changed, 139 insertions(+), 63 deletions(-) diff --git a/build_utils.py b/build_utils.py index 726a6282044..f5ccacae26f 100644 --- a/build_utils.py +++ b/build_utils.py @@ -21,6 +21,9 @@ from typing import Optional +SM_FAMILY_ORDER = ("sm9x", "sm10x", "sm12x") + + def get_git_version(cwd: Optional[Path] = None) -> str: """ Get git commit hash. @@ -44,3 +47,44 @@ def get_git_version(cwd: Optional[Path] = None) -> str: return git_version except Exception: return "unknown" + + +def sm_family_for_capability(major: int, minor: int) -> str: + if major < 10: + return "sm9x" + if major < 12: + return "sm10x" + return "sm12x" + + +def parse_cuda_arch_entry(entry: str) -> tuple[int, int]: + """Parse arch entries like '9.0a', '90', or '120f' into (major, minor).""" + entry = entry.strip() + if "." in entry: + major_str, minor_str = entry.split(".", 1) + major_digits = "".join(c for c in major_str if c.isdigit()) + minor_digits = "".join(c for c in minor_str if c.isdigit()) + if not major_digits: + raise ValueError(f"Invalid CUDA arch entry: {entry!r}") + return int(major_digits), int(minor_digits) if minor_digits else 0 + + digits = "".join(c for c in entry if c.isdigit()) + if not digits: + raise ValueError(f"Invalid CUDA arch entry: {entry!r}") + if len(digits) >= 2: + return int(digits[:-1]), int(digits[-1]) + return int(digits), 0 + + +def filter_arch_list_for_sm_family(arch_list: str, family: str) -> str: + """Filter a space-separated FLASHINFER_CUDA_ARCH_LIST for one SM family.""" + if family not in SM_FAMILY_ORDER: + raise ValueError( + f"Invalid SM family {family!r}; expected one of {SM_FAMILY_ORDER}" + ) + kept = [] + for entry in arch_list.split(): + major, minor = parse_cuda_arch_entry(entry) + if sm_family_for_capability(major, minor) == family: + kept.append(entry) + return " ".join(kept) diff --git a/flashinfer-jit-cache/build_backend.py b/flashinfer-jit-cache/build_backend.py index f79779d5c15..5c4e1705d6f 100644 --- a/flashinfer-jit-cache/build_backend.py +++ b/flashinfer-jit-cache/build_backend.py @@ -24,61 +24,25 @@ # Add parent directory to path to import flashinfer modules sys.path.insert(0, str(Path(__file__).parent.parent)) -from build_utils import get_git_version +from build_utils import ( + SM_FAMILY_ORDER, + filter_arch_list_for_sm_family, + get_git_version, +) # Skip version check when building flashinfer-jit-cache package os.environ["FLASHINFER_DISABLE_VERSION_CHECK"] = "1" -# SM family → arch-list filter predicate `(major, minor_int) -> bool`, applied to entries -# in FLASHINFER_CUDA_ARCH_LIST. The local-version suffix encodes the family so users (and -# pip) can resolve the right wheel: e.g. "0.6.11+cu130.sm10x". -# -# Keep in sync with `_sm_family_for_capability` in flashinfer/__main__.py. -def _is_sm9x(major, minor): # 7.5 / 8.0 / 8.9 / 9.0a — Ampere/Ada/Hopper - return major < 10 - - -def _is_sm10x(major, minor): # 10.0a / 10.3a / 11.0a — Datacenter Blackwell - return 10 <= major < 12 - - -def _is_sm12x(major, minor): # 12.0f / 12.1a — Consumer Blackwell - return major >= 12 - - -SM_FAMILIES = { - "sm9x": _is_sm9x, - "sm10x": _is_sm10x, - "sm12x": _is_sm12x, -} - - -def _filter_arch_list_for_family(arch_list: str, family: str) -> str: - """Filter a space-separated FLASHINFER_CUDA_ARCH_LIST to only entries belonging to `family`.""" - predicate = SM_FAMILIES[family] - kept = [] - for entry in arch_list.split(): - major_str, minor_str = entry.split(".", 1) - major = int(major_str) - # `minor_str` may carry a suffix like 'a' or 'f' — keep the whole string for output, - # but parse leading digits for the comparison. - leading_digits = "".join(c for c in minor_str if c.isdigit()) - minor = int(leading_digits) if leading_digits else 0 - if predicate(major, minor): - kept.append(entry) - return " ".join(kept) - - def _resolve_sm_family() -> str: """Return the SM family this build targets, or '' for a legacy multi-family build.""" family = os.environ.get("FLASHINFER_JIT_CACHE_SM_FAMILY", "").strip().lower() if not family: return "" - if family not in SM_FAMILIES: + if family not in SM_FAMILY_ORDER: raise RuntimeError( f"Invalid FLASHINFER_JIT_CACHE_SM_FAMILY={family!r}. " - f"Expected one of: {sorted(SM_FAMILIES)}" + f"Expected one of: {SM_FAMILY_ORDER}" ) return family @@ -95,7 +59,7 @@ def _apply_sm_family_filter() -> str: # we let it raise there to keep error messages consistent. return family - filtered = _filter_arch_list_for_family(arch_list, family) + filtered = filter_arch_list_for_sm_family(arch_list, family) if not filtered: raise RuntimeError( f"FLASHINFER_JIT_CACHE_SM_FAMILY={family} but FLASHINFER_CUDA_ARCH_LIST=" diff --git a/flashinfer/__main__.py b/flashinfer/__main__.py index d0f9c127a0a..ca46ad22e1c 100644 --- a/flashinfer/__main__.py +++ b/flashinfer/__main__.py @@ -26,6 +26,8 @@ from packaging.version import InvalidVersion, Version from tabulate import tabulate # type: ignore[import-untyped] +from build_utils import SM_FAMILY_ORDER, sm_family_for_capability + from .artifacts import ( ArtifactPath, download_artifacts, @@ -70,21 +72,6 @@ def _ensure_modules_registered(): return statuses -# SM family detection. Each family corresponds to a separately-built -# `flashinfer-jit-cache` wheel; the family is encoded in the local-version -# suffix (e.g. "0.6.11+cu130.sm10x"). Keep in sync with `SM_FAMILIES` in -# `flashinfer-jit-cache/build_backend.py`. -_SM_FAMILY_ORDER = ("sm9x", "sm10x", "sm12x") - - -def _sm_family_for_capability(major: int, minor: int) -> str: - if major < 10: - return "sm9x" # Ampere/Ada/Hopper - if major < 12: - return "sm10x" # Datacenter Blackwell (sm100/sm103/sm110) - return "sm12x" # Consumer Blackwell (sm120/sm121) - - def _detect_sm_family() -> str: """Return the SM family of the first visible CUDA device, or raise ClickException. @@ -105,7 +92,7 @@ def _detect_sm_family() -> str: ) major, minor = torch.cuda.get_device_capability(0) - return _sm_family_for_capability(major, minor) + return sm_family_for_capability(major, minor) @click.group(invoke_without_command=True) @@ -288,7 +275,11 @@ def _cuda_index_label(cuda_version: Version) -> str: def _build_jit_cache_requirement( - flashinfer_version: str, cuda_index_label: str, sm_family: str + flashinfer_version: str, + cuda_index_label: str, + sm_family: str, + *, + nightly: bool = False, ) -> str: if flashinfer_version == "0.0.0+unknown": raise click.ClickException( @@ -301,6 +292,13 @@ def _build_jit_cache_requirement( raise click.ClickException( f"Invalid FlashInfer version '{flashinfer_version}'." ) from e + if nightly and parsed.dev is None: + raise click.ClickException( + "Cannot infer a nightly flashinfer-jit-cache wheel from non-dev " + f"FlashInfer version '{flashinfer_version}'. Install flashinfer-python " + "from the nightly index or pass --flashinfer-version with a dev " + "version such as '0.6.11.devYYYYMMDD'." + ) return f"flashinfer-jit-cache=={parsed.public}+{cuda_index_label}.{sm_family}" @@ -365,7 +363,7 @@ def _build_package_install_command( @click.option( "--sm-family", default=None, - type=click.Choice(_SM_FAMILY_ORDER), + type=click.Choice(SM_FAMILY_ORDER), help=( "Override SM family detection. sm9x = Ampere/Ada/Hopper (≤sm90), " "sm10x = Datacenter Blackwell (sm100/103/110), sm12x = Consumer Blackwell (sm120/121)." @@ -406,7 +404,7 @@ def install_jit_cache_wheel_cmd( resolved_family = sm_family or _detect_sm_family() resolved_flashinfer_version = _resolve_flashinfer_version(flashinfer_version) requirement = _build_jit_cache_requirement( - resolved_flashinfer_version, cuda_label, resolved_family + resolved_flashinfer_version, cuda_label, resolved_family, nightly=nightly ) resolved_index_url = index_url or _build_jit_cache_index_url(cuda_label, nightly) diff --git a/tests/cli/test_cli_cmds.py b/tests/cli/test_cli_cmds.py index 179ad72c737..8680321fb20 100644 --- a/tests/cli/test_cli_cmds.py +++ b/tests/cli/test_cli_cmds.py @@ -246,6 +246,76 @@ def mock_build_install_command(index_url, requirement, nightly): ) +def test_install_jit_cache_wheel_dry_run_accepts_nightly_dev_version(monkeypatch): + def mock_build_install_command(index_url, requirement, nightly): + cmd = ["python", "-m", "pip", "install"] + if nightly: + cmd.append("--pre") + cmd.extend(["--index-url", index_url, requirement]) + return cmd + + monkeypatch.setattr( + "flashinfer.__main__._build_package_install_command", + mock_build_install_command, + ) + + out = _test_cmd_helper( + [ + "install-jit-cache-wheel", + "--nightly", + "--flashinfer-version", + "0.6.11.dev20260508+local", + "--cuda-version", + "cu130", + "--sm-family", + "sm12x", + "--dry-run", + ] + ) + + _assert_output_contains_all( + out, + "Wheel index: https://flashinfer.ai/whl/nightly/cu130", + "Requirement: flashinfer-jit-cache==0.6.11.dev20260508+cu130.sm12x", + "--pre --index-url https://flashinfer.ai/whl/nightly/cu130", + ) + + +def test_install_jit_cache_wheel_nightly_rejects_release_version(): + from click.testing import CliRunner + + from flashinfer.__main__ import cli + + runner = CliRunner() + result = runner.invoke( + cli, + [ + "install-jit-cache-wheel", + "--nightly", + "--flashinfer-version", + "0.6.11", + "--cuda-version", + "cu130", + "--sm-family", + "sm12x", + "--dry-run", + ], + ) + + assert result.exit_code != 0 + assert "Cannot infer a nightly flashinfer-jit-cache wheel" in result.output + + +def test_filter_arch_list_for_sm_family_accepts_integer_arch_entries(): + from build_utils import filter_arch_list_for_sm_family + + arch_list = "80 89 90 90a 100 103 110 120 12.1a" + + assert filter_arch_list_for_sm_family(arch_list, "sm9x") == "80 89 90 90a" + assert filter_arch_list_for_sm_family(arch_list, "sm10x") == "100 103 110" + assert filter_arch_list_for_sm_family(arch_list, "sm12x") == "120 12.1a" + + def test_resolve_flashinfer_version_uses_version_txt(monkeypatch, tmp_path): from flashinfer import __main__ as flashinfer_cli From dafe452df78946f396e5cc054c843442d9ae24da Mon Sep 17 00:00:00 2001 From: dierksen Date: Fri, 8 May 2026 15:22:56 -0500 Subject: [PATCH 5/8] Handle heterogeneous jit-cache wheel coverage --- .github/workflows/nightly-release.yml | 2 +- .github/workflows/release.yml | 2 +- README.md | 6 +- build_utils.py | 88 ++++++++++++++++- docs/installation.rst | 10 +- flashinfer/__main__.py | 20 +++- flashinfer/jit/env.py | 86 +++++++++++++--- ...ask_test_jit_cache_package_build_import.sh | 50 ++++------ tests/cli/test_cli_cmds.py | 99 ++++++++++++++++++- 9 files changed, 302 insertions(+), 61 deletions(-) diff --git a/.github/workflows/nightly-release.yml b/.github/workflows/nightly-release.yml index 665ab1ee2c0..8dfc07c8121 100644 --- a/.github/workflows/nightly-release.yml +++ b/.github/workflows/nightly-release.yml @@ -155,7 +155,7 @@ jobs: - name: Build wheel in container env: DOCKER_IMAGE: ${{ matrix.arch == 'aarch64' && format('pytorch/manylinuxaarch64-builder:cuda{0}', matrix.cuda) || format('pytorch/manylinux2_28-builder:cuda{0}', matrix.cuda) }} - FLASHINFER_CUDA_ARCH_LIST: ${{ matrix.cuda < '12.9' && '7.5 8.0 8.9 9.0a 10.0a 12.0a' || (matrix.cuda < '13.0' && '7.5 8.0 8.9 9.0a 10.0a 10.3a 12.0f' || '7.5 8.0 8.9 9.0a 10.0a 10.3a 11.0a 12.0f') }} + FLASHINFER_CUDA_ARCH_LIST: ${{ matrix.cuda < '12.9' && '7.5 8.0 8.9 9.0a 10.0a 12.0a' || (matrix.cuda < '13.0' && '7.5 8.0 8.9 9.0a 10.0a 10.3a 12.0f 12.1a' || '7.5 8.0 8.9 9.0a 10.0a 10.3a 11.0a 12.0f 12.1a') }} FLASHINFER_DEV_RELEASE_SUFFIX: ${{ needs.setup.outputs.dev_suffix }} run: | # Extract CUDA major and minor versions diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index c19ecc3c4ed..bea1d4b1a57 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -204,7 +204,7 @@ jobs: - name: Build wheel in container env: DOCKER_IMAGE: ${{ matrix.arch == 'aarch64' && format('pytorch/manylinuxaarch64-builder:cuda{0}', matrix.cuda) || format('pytorch/manylinux2_28-builder:cuda{0}', matrix.cuda) }} - FLASHINFER_CUDA_ARCH_LIST: ${{ matrix.cuda < '12.9' && '7.5 8.0 8.9 9.0a 10.0a 12.0a' || (matrix.cuda < '13.0' && '7.5 8.0 8.9 9.0a 10.0a 10.3a 12.0f' || '7.5 8.0 8.9 9.0a 10.0a 10.3a 11.0a 12.0f') }} + FLASHINFER_CUDA_ARCH_LIST: ${{ matrix.cuda < '12.9' && '7.5 8.0 8.9 9.0a 10.0a 12.0a' || (matrix.cuda < '13.0' && '7.5 8.0 8.9 9.0a 10.0a 10.3a 12.0f 12.1a' || '7.5 8.0 8.9 9.0a 10.0a 10.3a 11.0a 12.0f 12.1a') }} run: | # Extract CUDA major and minor versions CUDA_MAJOR=$(echo "${{ matrix.cuda }}" | cut -d'.' -f1) diff --git a/README.md b/README.md index 68241f5ed89..78b79f30027 100644 --- a/README.md +++ b/README.md @@ -107,7 +107,7 @@ pip install flashinfer-python flashinfer-cubin flashinfer install-jit-cache-wheel ``` -`flashinfer-jit-cache` is published as separate per-(CUDA, SM family) wheels — `sm9x` (Ampere/Ada/Hopper, ≤sm90), `sm10x` (Datacenter Blackwell, sm100/103/110), and `sm12x` (Consumer Blackwell, sm120/121) — because a single multi-arch wheel exceeds GitHub Releases' 2 GiB asset limit. The CLI resolves the right one for you. +`flashinfer-jit-cache` is published as separate per-(CUDA, SM family) wheels — `sm9x` (Ampere/Ada/Hopper, ≤sm90), `sm10x` (Datacenter Blackwell, sm100/103/110), and `sm12x` (Consumer Blackwell, sm120/121) — because a single multi-arch wheel exceeds GitHub Releases' 2 GiB asset limit. The CLI checks all visible GPUs and resolves the right one for you when one wheel can cover the system. Blackwell-family wheels also retain the `sm80` base arch alongside their native Blackwell archs. **For Blackwell (SM100+) CuTe DSL kernels**, install with the CUDA 13 extra to enable Blackwell-optimized kernels: @@ -167,14 +167,14 @@ python -m pip install dist/*.whl ```bash # flashinfer-jit-cache (customize for your target GPUs) -export FLASHINFER_CUDA_ARCH_LIST="7.5 8.0 8.9 9.0a 10.0a 10.3a 11.0a 12.0f" +export FLASHINFER_CUDA_ARCH_LIST="7.5 8.0 8.9 9.0a 10.0a 10.3a 11.0a 12.0f 12.1a" cd flashinfer-jit-cache python -m build --no-isolation --wheel python -m pip install dist/*.whl # Or build a per-SM-family wheel matching a release artifact (smaller; one GPU family). # The local-version on the resulting wheel will include the family (e.g. +cu130.sm10x). -export FLASHINFER_CUDA_ARCH_LIST="10.0a 10.3a 11.0a" +export FLASHINFER_CUDA_ARCH_LIST="8.0 10.0a 10.3a 11.0a" export FLASHINFER_JIT_CACHE_SM_FAMILY="sm10x" python -m build --no-isolation --wheel ``` diff --git a/build_utils.py b/build_utils.py index f5ccacae26f..660454d75a1 100644 --- a/build_utils.py +++ b/build_utils.py @@ -22,6 +22,10 @@ SM_FAMILY_ORDER = ("sm9x", "sm10x", "sm12x") +SM_FAMILY_BASE_ARCHS = { + "sm10x": ("8.0",), + "sm12x": ("8.0",), +} def get_git_version(cwd: Optional[Path] = None) -> str: @@ -57,6 +61,58 @@ def sm_family_for_capability(major: int, minor: int) -> str: return "sm12x" +def jit_cache_sm_family_for_capabilities(capabilities: list[tuple[int, int]]) -> str: + """Return the jit-cache wheel family covering the visible GPU capabilities.""" + if not capabilities: + raise ValueError("No CUDA device capabilities were provided.") + + native_families = { + sm_family_for_capability(major, minor) for major, minor in capabilities + } + if len(native_families) == 1: + return next(iter(native_families)) + + for family, base_archs in SM_FAMILY_BASE_ARCHS.items(): + base_caps = {parse_cuda_arch_entry(entry) for entry in base_archs} + has_native_arch = any( + sm_family_for_capability(major, minor) == family + for major, minor in capabilities + ) + if has_native_arch and all( + sm_family_for_capability(major, minor) == family + or (major, minor) in base_caps + for major, minor in capabilities + ): + return family + + details = ", ".join( + f"sm{major}{minor} ({sm_family_for_capability(major, minor)})" + for major, minor in capabilities + ) + raise ValueError( + "Visible CUDA devices are not covered by a single " + f"flashinfer-jit-cache SM-family wheel: {details}." + ) + + +def jit_cache_sm_family_covers_capabilities( + family: str, capabilities: list[tuple[int, int]] +) -> bool: + """Return whether a jit-cache wheel family contains all requested capabilities.""" + if family not in SM_FAMILY_ORDER: + raise ValueError( + f"Invalid SM family {family!r}; expected one of {SM_FAMILY_ORDER}" + ) + + base_caps = { + parse_cuda_arch_entry(entry) for entry in SM_FAMILY_BASE_ARCHS.get(family, ()) + } + return all( + sm_family_for_capability(major, minor) == family or (major, minor) in base_caps + for major, minor in capabilities + ) + + def parse_cuda_arch_entry(entry: str) -> tuple[int, int]: """Parse arch entries like '9.0a', '90', or '120f' into (major, minor).""" entry = entry.strip() @@ -77,14 +133,38 @@ def parse_cuda_arch_entry(entry: str) -> tuple[int, int]: def filter_arch_list_for_sm_family(arch_list: str, family: str) -> str: - """Filter a space-separated FLASHINFER_CUDA_ARCH_LIST for one SM family.""" + """Filter FLASHINFER_CUDA_ARCH_LIST for one jit-cache wheel family.""" if family not in SM_FAMILY_ORDER: raise ValueError( f"Invalid SM family {family!r}; expected one of {SM_FAMILY_ORDER}" ) - kept = [] + + parsed_entries = [] for entry in arch_list.split(): major, minor = parse_cuda_arch_entry(entry) + parsed_entries.append((entry, major, minor)) + + native_entries = [] + for entry, major, minor in parsed_entries: if sm_family_for_capability(major, minor) == family: - kept.append(entry) - return " ".join(kept) + native_entries.append(entry) + + if not native_entries: + return "" + + kept = [] + existing_by_capability = { + (major, minor): entry for entry, major, minor in parsed_entries + } + for base_arch in SM_FAMILY_BASE_ARCHS.get(family, ()): + base_capability = parse_cuda_arch_entry(base_arch) + kept.append(existing_by_capability.get(base_capability, base_arch)) + kept.extend(native_entries) + + seen = set() + kept_unique = [] + for entry in kept: + if entry not in seen: + seen.add(entry) + kept_unique.append(entry) + return " ".join(kept_unique) diff --git a/docs/installation.rst b/docs/installation.rst index df52d48973a..f027c364721 100644 --- a/docs/installation.rst +++ b/docs/installation.rst @@ -50,7 +50,8 @@ This eliminates compilation and downloading overhead at runtime. ``flashinfer-jit-cache`` is published as separate per-(CUDA, SM family) wheels because a single multi-arch wheel exceeds GitHub Releases' 2 GiB asset limit. The CLI resolves -the right one for you. To pick manually, override the autodetection: +the right one for you when all visible GPUs can be covered by one wheel. To pick +manually, override the autodetection: .. code-block:: bash @@ -61,7 +62,8 @@ the right one for you. To pick manually, override the autodetection: flashinfer install-jit-cache-wheel --dry-run The SM families are: ``sm9x`` (Ampere/Ada/Hopper, ≤sm90), ``sm10x`` (Datacenter Blackwell, -sm100/103/110), and ``sm12x`` (Consumer Blackwell, sm120/121). +sm100/103/110), and ``sm12x`` (Consumer Blackwell, sm120/121). Blackwell-family +wheels also retain the ``sm80`` base arch alongside their native Blackwell archs. .. _install-from-source: @@ -122,7 +124,7 @@ You can follow the steps below to install FlashInfer from source code: .. code-block:: bash - export FLASHINFER_CUDA_ARCH_LIST="7.5 8.0 8.9 9.0a 10.0a 10.3a 11.0a 12.0f" + export FLASHINFER_CUDA_ARCH_LIST="7.5 8.0 8.9 9.0a 10.0a 10.3a 11.0a 12.0f 12.1a" cd flashinfer-jit-cache python -m build --no-isolation --wheel python -m pip install dist/*.whl @@ -133,7 +135,7 @@ You can follow the steps below to install FlashInfer from source code: .. code-block:: bash - export FLASHINFER_CUDA_ARCH_LIST="10.0a 10.3a 11.0a" + export FLASHINFER_CUDA_ARCH_LIST="8.0 10.0a 10.3a 11.0a" export FLASHINFER_JIT_CACHE_SM_FAMILY="sm10x" cd flashinfer-jit-cache python -m build --no-isolation --wheel diff --git a/flashinfer/__main__.py b/flashinfer/__main__.py index ca46ad22e1c..85501d59b4f 100644 --- a/flashinfer/__main__.py +++ b/flashinfer/__main__.py @@ -26,7 +26,7 @@ from packaging.version import InvalidVersion, Version from tabulate import tabulate # type: ignore[import-untyped] -from build_utils import SM_FAMILY_ORDER, sm_family_for_capability +from build_utils import SM_FAMILY_ORDER, jit_cache_sm_family_for_capabilities from .artifacts import ( ArtifactPath, @@ -73,7 +73,7 @@ def _ensure_modules_registered(): def _detect_sm_family() -> str: - """Return the SM family of the first visible CUDA device, or raise ClickException. + """Return the SM family covering all visible CUDA devices, or raise ClickException. Used as the default for `flashinfer install-jit-cache-wheel`. """ @@ -91,8 +91,18 @@ def _detect_sm_family() -> str: "Pass --sm-family {sm9x|sm10x|sm12x} or run on a host with the target GPU." ) - major, minor = torch.cuda.get_device_capability(0) - return sm_family_for_capability(major, minor) + capabilities = [ + torch.cuda.get_device_capability(device) + for device in range(torch.cuda.device_count()) + ] + try: + return jit_cache_sm_family_for_capabilities(capabilities) + except ValueError as e: + raise click.ClickException( + f"{e} Pass --sm-family explicitly if you only want to install a wheel " + "for one GPU family, or build flashinfer-jit-cache from source with " + "FLASHINFER_CUDA_ARCH_LIST covering all target GPUs." + ) from e @click.group(invoke_without_command=True) @@ -394,7 +404,7 @@ def install_jit_cache_wheel_cmd( ): """Install the matching flashinfer-jit-cache wheel for this GPU + CUDA. - Detects FlashInfer version, CUDA version, and GPU compute capability, then runs + Detects FlashInfer version, CUDA version, and visible GPU capabilities, then runs `pip install flashinfer-jit-cache==+cu.` against the FlashInfer wheel index. Use --sm-family to bypass GPU detection (e.g. when prepping a container image on a CPU-only host). diff --git a/flashinfer/jit/env.py b/flashinfer/jit/env.py index e79c03c6e23..051bea2d678 100644 --- a/flashinfer/jit/env.py +++ b/flashinfer/jit/env.py @@ -20,6 +20,14 @@ import os import pathlib +from typing import Optional + +from build_utils import ( + SM_FAMILY_ORDER, + jit_cache_sm_family_covers_capabilities, + jit_cache_sm_family_for_capabilities, +) + from ..compilation_context import CompilationContext from ..version import __version__ as flashinfer_version from packaging.version import InvalidVersion, Version @@ -32,6 +40,61 @@ def _public_package_version(version: str) -> str: return version.split("+", 1)[0] +def _sm_family_from_package_version(version: str) -> Optional[str]: + try: + local_version = Version(version).local + except InvalidVersion: + local_version = version.split("+", 1)[1] if "+" in version else None + if not local_version: + return None + for part in local_version.split("."): + if part in SM_FAMILY_ORDER: + return part + return None + + +def _visible_cuda_device_capabilities() -> list[tuple[int, int]]: + try: + import torch + except Exception: + return [] + + if not torch.cuda.is_available() or torch.cuda.device_count() == 0: + return [] + return [ + torch.cuda.get_device_capability(device) + for device in range(torch.cuda.device_count()) + ] + + +def _validate_jit_cache_sm_family(flashinfer_jit_cache_version: str) -> None: + installed_family = _sm_family_from_package_version(flashinfer_jit_cache_version) + if installed_family is None: + return + + capabilities = _visible_cuda_device_capabilities() + if not capabilities: + return + + if not jit_cache_sm_family_covers_capabilities(installed_family, capabilities): + try: + expected_family = jit_cache_sm_family_for_capabilities(capabilities) + except ValueError as e: + raise RuntimeError( + "Visible CUDA devices cannot be covered by a single " + f"flashinfer-jit-cache wheel: {e}" + ) from e + capability_text = ", ".join( + f"sm{major}{minor}" for major, minor in capabilities + ) + raise RuntimeError( + f"flashinfer-jit-cache SM family ({installed_family}) does not match " + f"visible CUDA devices ({capability_text}; expected {expected_family}). " + "Install the matching flashinfer-jit-cache wheel or set " + "FLASHINFER_DISABLE_VERSION_CHECK=1 to bypass this check." + ) + + def has_flashinfer_jit_cache() -> bool: """ Check if flashinfer_jit_cache module is available. @@ -120,18 +183,17 @@ def _get_aot_dir(): # NOTE(Zihao): we don't use exact version match here because the version of flashinfer-jit-cache # contains the CUDA version suffix: e.g. 0.3.1+cu129. # Allow bypassing version check with environment variable - if ( - not os.getenv("FLASHINFER_DISABLE_VERSION_CHECK") - and flashinfer_version != "0.0.0+unknown" - and _public_package_version(flashinfer_version) - != _public_package_version(flashinfer_jit_cache_version) - ): - raise RuntimeError( - f"flashinfer-jit-cache version ({flashinfer_jit_cache_version}) does not match " - f"flashinfer version ({flashinfer_version}). " - "Please install the same version of both packages. " - "Set FLASHINFER_DISABLE_VERSION_CHECK=1 to bypass this check." - ) + if not os.getenv("FLASHINFER_DISABLE_VERSION_CHECK"): + if flashinfer_version != "0.0.0+unknown" and _public_package_version( + flashinfer_version + ) != _public_package_version(flashinfer_jit_cache_version): + raise RuntimeError( + f"flashinfer-jit-cache version ({flashinfer_jit_cache_version}) does not match " + f"flashinfer version ({flashinfer_version}). " + "Please install the same version of both packages. " + "Set FLASHINFER_DISABLE_VERSION_CHECK=1 to bypass this check." + ) + _validate_jit_cache_sm_family(flashinfer_jit_cache_version) return pathlib.Path(flashinfer_jit_cache.get_jit_cache_dir()) diff --git a/scripts/task_test_jit_cache_package_build_import.sh b/scripts/task_test_jit_cache_package_build_import.sh index c6bb3fe4a46..2e43b97b6a3 100755 --- a/scripts/task_test_jit_cache_package_build_import.sh +++ b/scripts/task_test_jit_cache_package_build_import.sh @@ -145,10 +145,12 @@ if cuda_ver is not None: arches.append("10.3a") arches.append("11.0a") arches.append("12.0f") + arches.append("12.1a") elif (major, minor) >= (12, 9): arches.append("10.0a") arches.append("10.3a") arches.append("12.0f") + arches.append("12.1a") elif (major, minor) >= (12, 8): arches.append("10.0a") arches.append("12.0a") @@ -166,36 +168,26 @@ echo "FLASHINFER_CUDA_ARCH_LIST: ${FLASHINFER_CUDA_ARCH_LIST}" # Keep in sync with SM_FAMILIES in flashinfer-jit-cache/build_backend.py. if [ -n "${FLASHINFER_JIT_CACHE_SM_FAMILY}" ]; then echo "Filtering arch list for FLASHINFER_JIT_CACHE_SM_FAMILY=${FLASHINFER_JIT_CACHE_SM_FAMILY}" - FAMILY="${FLASHINFER_JIT_CACHE_SM_FAMILY}" - case "${FAMILY}" in - sm9x|sm10x|sm12x) ;; - *) echo "ERROR: invalid FLASHINFER_JIT_CACHE_SM_FAMILY: ${FAMILY}"; exit 1;; - esac - FILTERED="" - for entry in ${FLASHINFER_CUDA_ARCH_LIST}; do - major="${entry%%.*}" - keep=false - case "${FAMILY}" in - sm9x) - if [ "${major}" -lt 10 ]; then keep=true; fi - ;; - sm10x) - if [ "${major}" -ge 10 ] && [ "${major}" -lt 12 ]; then keep=true; fi - ;; - sm12x) - if [ "${major}" -ge 12 ]; then keep=true; fi - ;; - esac - if [ "${keep}" = true ]; then - if [ -z "${FILTERED}" ]; then - FILTERED="${entry}" - else - FILTERED="${FILTERED} ${entry}" - fi - fi - done + FILTERED=$(python3 - <<'PY' +import os +import sys + +from build_utils import filter_arch_list_for_sm_family + +try: + print( + filter_arch_list_for_sm_family( + os.environ["FLASHINFER_CUDA_ARCH_LIST"], + os.environ["FLASHINFER_JIT_CACHE_SM_FAMILY"], + ) + ) +except ValueError as e: + print(f"ERROR: {e}", file=sys.stderr) + sys.exit(1) +PY +) if [ -z "${FILTERED}" ]; then - echo "ERROR: family ${FAMILY} has no matching archs in '${FLASHINFER_CUDA_ARCH_LIST}'." + echo "ERROR: family ${FLASHINFER_JIT_CACHE_SM_FAMILY} has no matching native archs in '${FLASHINFER_CUDA_ARCH_LIST}'." echo " This (CUDA, family) combination should be excluded in the workflow matrix." exit 1 fi diff --git a/tests/cli/test_cli_cmds.py b/tests/cli/test_cli_cmds.py index 8680321fb20..c5ca7f8ad71 100644 --- a/tests/cli/test_cli_cmds.py +++ b/tests/cli/test_cli_cmds.py @@ -13,6 +13,11 @@ the other logging/replay functionality tests, since it's tightly coupled with that feature. """ +import sys +import types + +import pytest + from .cli_cmd_helpers import ( _test_cmd_helper, _assert_output_contains_all, @@ -306,14 +311,81 @@ def test_install_jit_cache_wheel_nightly_rejects_release_version(): assert "Cannot infer a nightly flashinfer-jit-cache wheel" in result.output +def test_detect_sm_family_checks_all_visible_gpus(monkeypatch): + from flashinfer import __main__ as flashinfer_cli + + class FakeCuda: + def __init__(self, capabilities): + self._capabilities = capabilities + + def is_available(self): + return True + + def device_count(self): + return len(self._capabilities) + + def get_device_capability(self, device): + return self._capabilities[device] + + monkeypatch.setitem( + sys.modules, + "torch", + types.SimpleNamespace(cuda=FakeCuda([(8, 0), (12, 1)])), + ) + + assert flashinfer_cli._detect_sm_family() == "sm12x" + + +def test_detect_sm_family_rejects_unsupported_heterogeneous_gpus(monkeypatch): + import click + + from flashinfer import __main__ as flashinfer_cli + + class FakeCuda: + def is_available(self): + return True + + def device_count(self): + return 2 + + def get_device_capability(self, device): + return [(9, 0), (12, 1)][device] + + monkeypatch.setitem(sys.modules, "torch", types.SimpleNamespace(cuda=FakeCuda())) + + with pytest.raises(click.ClickException, match="single flashinfer-jit-cache"): + flashinfer_cli._detect_sm_family() + + def test_filter_arch_list_for_sm_family_accepts_integer_arch_entries(): from build_utils import filter_arch_list_for_sm_family arch_list = "80 89 90 90a 100 103 110 120 12.1a" assert filter_arch_list_for_sm_family(arch_list, "sm9x") == "80 89 90 90a" - assert filter_arch_list_for_sm_family(arch_list, "sm10x") == "100 103 110" - assert filter_arch_list_for_sm_family(arch_list, "sm12x") == "120 12.1a" + assert filter_arch_list_for_sm_family(arch_list, "sm10x") == "80 100 103 110" + assert filter_arch_list_for_sm_family(arch_list, "sm12x") == "80 120 12.1a" + + +def test_filter_arch_list_for_sm_family_adds_sm80_base_arch(): + from build_utils import filter_arch_list_for_sm_family + + assert filter_arch_list_for_sm_family("10.0a", "sm10x") == "8.0 10.0a" + assert filter_arch_list_for_sm_family("12.1a", "sm12x") == "8.0 12.1a" + assert filter_arch_list_for_sm_family("8.0 8.9", "sm12x") == "" + + +def test_jit_cache_sm_family_for_capabilities_allows_sm80_base_arch(): + from build_utils import ( + jit_cache_sm_family_covers_capabilities, + jit_cache_sm_family_for_capabilities, + ) + + assert jit_cache_sm_family_for_capabilities([(8, 0), (10, 3)]) == "sm10x" + assert jit_cache_sm_family_for_capabilities([(8, 0), (12, 1)]) == "sm12x" + assert jit_cache_sm_family_covers_capabilities("sm12x", [(8, 0)]) + with pytest.raises(ValueError, match="single flashinfer-jit-cache"): + jit_cache_sm_family_for_capabilities([(9, 0), (12, 1)]) def test_resolve_flashinfer_version_uses_version_txt(monkeypatch, tmp_path): @@ -368,6 +440,29 @@ def test_public_package_version_ignores_local_suffix(): ) +def test_sm_family_from_package_version(): + from flashinfer.jit.env import _sm_family_from_package_version + + assert _sm_family_from_package_version("0.6.11+cu130.sm12x") == "sm12x" + assert _sm_family_from_package_version("0.6.11.dev20260508+cu130.sm10x") == ( + "sm10x" + ) + assert _sm_family_from_package_version("0.6.11+cu130") is None + + +def test_validate_jit_cache_sm_family_rejects_wrong_family(monkeypatch): + from flashinfer.jit import env as jit_env + + monkeypatch.setattr(jit_env, "_visible_cuda_device_capabilities", lambda: [(12, 1)]) + + jit_env._validate_jit_cache_sm_family("0.6.11+cu130.sm12x") + with pytest.raises(RuntimeError, match="does not match visible CUDA devices"): + jit_env._validate_jit_cache_sm_family("0.6.11+cu130.sm9x") + + monkeypatch.setattr(jit_env, "_visible_cuda_device_capabilities", lambda: [(8, 0)]) + jit_env._validate_jit_cache_sm_family("0.6.11+cu130.sm12x") + + class MockJitSpec: """Mock JitSpec for testing export-compile-commands.""" From 1889027a283893ceeebe44ca063bac2725330807 Mon Sep 17 00:00:00 2001 From: dierksen Date: Fri, 8 May 2026 15:30:18 -0500 Subject: [PATCH 6/8] Use sm120f as sm12x family target --- .github/workflows/nightly-release.yml | 2 +- .github/workflows/release.yml | 2 +- README.md | 2 +- docs/installation.rst | 2 +- scripts/task_test_jit_cache_package_build_import.sh | 2 -- tests/cli/test_cli_cmds.py | 6 +++--- 6 files changed, 7 insertions(+), 9 deletions(-) diff --git a/.github/workflows/nightly-release.yml b/.github/workflows/nightly-release.yml index 8dfc07c8121..665ab1ee2c0 100644 --- a/.github/workflows/nightly-release.yml +++ b/.github/workflows/nightly-release.yml @@ -155,7 +155,7 @@ jobs: - name: Build wheel in container env: DOCKER_IMAGE: ${{ matrix.arch == 'aarch64' && format('pytorch/manylinuxaarch64-builder:cuda{0}', matrix.cuda) || format('pytorch/manylinux2_28-builder:cuda{0}', matrix.cuda) }} - FLASHINFER_CUDA_ARCH_LIST: ${{ matrix.cuda < '12.9' && '7.5 8.0 8.9 9.0a 10.0a 12.0a' || (matrix.cuda < '13.0' && '7.5 8.0 8.9 9.0a 10.0a 10.3a 12.0f 12.1a' || '7.5 8.0 8.9 9.0a 10.0a 10.3a 11.0a 12.0f 12.1a') }} + FLASHINFER_CUDA_ARCH_LIST: ${{ matrix.cuda < '12.9' && '7.5 8.0 8.9 9.0a 10.0a 12.0a' || (matrix.cuda < '13.0' && '7.5 8.0 8.9 9.0a 10.0a 10.3a 12.0f' || '7.5 8.0 8.9 9.0a 10.0a 10.3a 11.0a 12.0f') }} FLASHINFER_DEV_RELEASE_SUFFIX: ${{ needs.setup.outputs.dev_suffix }} run: | # Extract CUDA major and minor versions diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index bea1d4b1a57..c19ecc3c4ed 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -204,7 +204,7 @@ jobs: - name: Build wheel in container env: DOCKER_IMAGE: ${{ matrix.arch == 'aarch64' && format('pytorch/manylinuxaarch64-builder:cuda{0}', matrix.cuda) || format('pytorch/manylinux2_28-builder:cuda{0}', matrix.cuda) }} - FLASHINFER_CUDA_ARCH_LIST: ${{ matrix.cuda < '12.9' && '7.5 8.0 8.9 9.0a 10.0a 12.0a' || (matrix.cuda < '13.0' && '7.5 8.0 8.9 9.0a 10.0a 10.3a 12.0f 12.1a' || '7.5 8.0 8.9 9.0a 10.0a 10.3a 11.0a 12.0f 12.1a') }} + FLASHINFER_CUDA_ARCH_LIST: ${{ matrix.cuda < '12.9' && '7.5 8.0 8.9 9.0a 10.0a 12.0a' || (matrix.cuda < '13.0' && '7.5 8.0 8.9 9.0a 10.0a 10.3a 12.0f' || '7.5 8.0 8.9 9.0a 10.0a 10.3a 11.0a 12.0f') }} run: | # Extract CUDA major and minor versions CUDA_MAJOR=$(echo "${{ matrix.cuda }}" | cut -d'.' -f1) diff --git a/README.md b/README.md index 78b79f30027..690f7d6ab75 100644 --- a/README.md +++ b/README.md @@ -167,7 +167,7 @@ python -m pip install dist/*.whl ```bash # flashinfer-jit-cache (customize for your target GPUs) -export FLASHINFER_CUDA_ARCH_LIST="7.5 8.0 8.9 9.0a 10.0a 10.3a 11.0a 12.0f 12.1a" +export FLASHINFER_CUDA_ARCH_LIST="7.5 8.0 8.9 9.0a 10.0a 10.3a 11.0a 12.0f" cd flashinfer-jit-cache python -m build --no-isolation --wheel python -m pip install dist/*.whl diff --git a/docs/installation.rst b/docs/installation.rst index f027c364721..baefc852d78 100644 --- a/docs/installation.rst +++ b/docs/installation.rst @@ -124,7 +124,7 @@ You can follow the steps below to install FlashInfer from source code: .. code-block:: bash - export FLASHINFER_CUDA_ARCH_LIST="7.5 8.0 8.9 9.0a 10.0a 10.3a 11.0a 12.0f 12.1a" + export FLASHINFER_CUDA_ARCH_LIST="7.5 8.0 8.9 9.0a 10.0a 10.3a 11.0a 12.0f" cd flashinfer-jit-cache python -m build --no-isolation --wheel python -m pip install dist/*.whl diff --git a/scripts/task_test_jit_cache_package_build_import.sh b/scripts/task_test_jit_cache_package_build_import.sh index 2e43b97b6a3..b500cdd9fff 100755 --- a/scripts/task_test_jit_cache_package_build_import.sh +++ b/scripts/task_test_jit_cache_package_build_import.sh @@ -145,12 +145,10 @@ if cuda_ver is not None: arches.append("10.3a") arches.append("11.0a") arches.append("12.0f") - arches.append("12.1a") elif (major, minor) >= (12, 9): arches.append("10.0a") arches.append("10.3a") arches.append("12.0f") - arches.append("12.1a") elif (major, minor) >= (12, 8): arches.append("10.0a") arches.append("12.0a") diff --git a/tests/cli/test_cli_cmds.py b/tests/cli/test_cli_cmds.py index c5ca7f8ad71..b0839941650 100644 --- a/tests/cli/test_cli_cmds.py +++ b/tests/cli/test_cli_cmds.py @@ -360,18 +360,18 @@ def get_device_capability(self, device): def test_filter_arch_list_for_sm_family_accepts_integer_arch_entries(): from build_utils import filter_arch_list_for_sm_family - arch_list = "80 89 90 90a 100 103 110 120 12.1a" + arch_list = "80 89 90 90a 100 103 110 120" assert filter_arch_list_for_sm_family(arch_list, "sm9x") == "80 89 90 90a" assert filter_arch_list_for_sm_family(arch_list, "sm10x") == "80 100 103 110" - assert filter_arch_list_for_sm_family(arch_list, "sm12x") == "80 120 12.1a" + assert filter_arch_list_for_sm_family(arch_list, "sm12x") == "80 120" def test_filter_arch_list_for_sm_family_adds_sm80_base_arch(): from build_utils import filter_arch_list_for_sm_family assert filter_arch_list_for_sm_family("10.0a", "sm10x") == "8.0 10.0a" - assert filter_arch_list_for_sm_family("12.1a", "sm12x") == "8.0 12.1a" + assert filter_arch_list_for_sm_family("12.0f", "sm12x") == "8.0 12.0f" assert filter_arch_list_for_sm_family("8.0 8.9", "sm12x") == "" From d3d39bbdc4ff9e4b0193b6577e9001af3906fb7c Mon Sep 17 00:00:00 2001 From: dierksen Date: Fri, 8 May 2026 16:00:36 -0500 Subject: [PATCH 7/8] Limit sm110 jit-cache builds to aarch64 --- .github/workflows/nightly-release.yml | 2 +- .github/workflows/release.yml | 2 +- README.md | 5 +++-- docs/installation.rst | 5 +++-- scripts/task_test_jit_cache_package_build_import.sh | 7 ++++++- 5 files changed, 14 insertions(+), 7 deletions(-) diff --git a/.github/workflows/nightly-release.yml b/.github/workflows/nightly-release.yml index 665ab1ee2c0..b73c323dc1d 100644 --- a/.github/workflows/nightly-release.yml +++ b/.github/workflows/nightly-release.yml @@ -155,7 +155,7 @@ jobs: - name: Build wheel in container env: DOCKER_IMAGE: ${{ matrix.arch == 'aarch64' && format('pytorch/manylinuxaarch64-builder:cuda{0}', matrix.cuda) || format('pytorch/manylinux2_28-builder:cuda{0}', matrix.cuda) }} - FLASHINFER_CUDA_ARCH_LIST: ${{ matrix.cuda < '12.9' && '7.5 8.0 8.9 9.0a 10.0a 12.0a' || (matrix.cuda < '13.0' && '7.5 8.0 8.9 9.0a 10.0a 10.3a 12.0f' || '7.5 8.0 8.9 9.0a 10.0a 10.3a 11.0a 12.0f') }} + FLASHINFER_CUDA_ARCH_LIST: ${{ matrix.cuda < '12.9' && '7.5 8.0 8.9 9.0a 10.0a 12.0a' || (matrix.cuda < '13.0' && '7.5 8.0 8.9 9.0a 10.0a 10.3a 12.0f' || (matrix.arch == 'aarch64' && '7.5 8.0 8.9 9.0a 10.0a 10.3a 11.0a 12.0f' || '7.5 8.0 8.9 9.0a 10.0a 10.3a 12.0f')) }} FLASHINFER_DEV_RELEASE_SUFFIX: ${{ needs.setup.outputs.dev_suffix }} run: | # Extract CUDA major and minor versions diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index c19ecc3c4ed..35e980ec8c3 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -204,7 +204,7 @@ jobs: - name: Build wheel in container env: DOCKER_IMAGE: ${{ matrix.arch == 'aarch64' && format('pytorch/manylinuxaarch64-builder:cuda{0}', matrix.cuda) || format('pytorch/manylinux2_28-builder:cuda{0}', matrix.cuda) }} - FLASHINFER_CUDA_ARCH_LIST: ${{ matrix.cuda < '12.9' && '7.5 8.0 8.9 9.0a 10.0a 12.0a' || (matrix.cuda < '13.0' && '7.5 8.0 8.9 9.0a 10.0a 10.3a 12.0f' || '7.5 8.0 8.9 9.0a 10.0a 10.3a 11.0a 12.0f') }} + FLASHINFER_CUDA_ARCH_LIST: ${{ matrix.cuda < '12.9' && '7.5 8.0 8.9 9.0a 10.0a 12.0a' || (matrix.cuda < '13.0' && '7.5 8.0 8.9 9.0a 10.0a 10.3a 12.0f' || (matrix.arch == 'aarch64' && '7.5 8.0 8.9 9.0a 10.0a 10.3a 11.0a 12.0f' || '7.5 8.0 8.9 9.0a 10.0a 10.3a 12.0f')) }} run: | # Extract CUDA major and minor versions CUDA_MAJOR=$(echo "${{ matrix.cuda }}" | cut -d'.' -f1) diff --git a/README.md b/README.md index 690f7d6ab75..60da2c8b89f 100644 --- a/README.md +++ b/README.md @@ -167,14 +167,15 @@ python -m pip install dist/*.whl ```bash # flashinfer-jit-cache (customize for your target GPUs) -export FLASHINFER_CUDA_ARCH_LIST="7.5 8.0 8.9 9.0a 10.0a 10.3a 11.0a 12.0f" +export FLASHINFER_CUDA_ARCH_LIST="7.5 8.0 8.9 9.0a 10.0a 10.3a 12.0f" +# Add 11.0a when building for Jetson AGX Thor / T5000 aarch64 targets. cd flashinfer-jit-cache python -m build --no-isolation --wheel python -m pip install dist/*.whl # Or build a per-SM-family wheel matching a release artifact (smaller; one GPU family). # The local-version on the resulting wheel will include the family (e.g. +cu130.sm10x). -export FLASHINFER_CUDA_ARCH_LIST="8.0 10.0a 10.3a 11.0a" +export FLASHINFER_CUDA_ARCH_LIST="8.0 10.0a 10.3a" export FLASHINFER_JIT_CACHE_SM_FAMILY="sm10x" python -m build --no-isolation --wheel ``` diff --git a/docs/installation.rst b/docs/installation.rst index baefc852d78..5863d9bfb29 100644 --- a/docs/installation.rst +++ b/docs/installation.rst @@ -124,7 +124,8 @@ You can follow the steps below to install FlashInfer from source code: .. code-block:: bash - export FLASHINFER_CUDA_ARCH_LIST="7.5 8.0 8.9 9.0a 10.0a 10.3a 11.0a 12.0f" + export FLASHINFER_CUDA_ARCH_LIST="7.5 8.0 8.9 9.0a 10.0a 10.3a 12.0f" + # Add 11.0a when building for Jetson AGX Thor / T5000 aarch64 targets. cd flashinfer-jit-cache python -m build --no-isolation --wheel python -m pip install dist/*.whl @@ -135,7 +136,7 @@ You can follow the steps below to install FlashInfer from source code: .. code-block:: bash - export FLASHINFER_CUDA_ARCH_LIST="8.0 10.0a 10.3a 11.0a" + export FLASHINFER_CUDA_ARCH_LIST="8.0 10.0a 10.3a" export FLASHINFER_JIT_CACHE_SM_FAMILY="sm10x" cd flashinfer-jit-cache python -m build --no-isolation --wheel diff --git a/scripts/task_test_jit_cache_package_build_import.sh b/scripts/task_test_jit_cache_package_build_import.sh index b500cdd9fff..0f30e7a5bbf 100755 --- a/scripts/task_test_jit_cache_package_build_import.sh +++ b/scripts/task_test_jit_cache_package_build_import.sh @@ -134,6 +134,9 @@ echo "Cache cleaned." echo "" echo "Detecting CUDA architecture list..." export FLASHINFER_CUDA_ARCH_LIST=$(python3 -c ' +import os +import platform + import torch cuda_ver = torch.version.cuda arches = ["7.5", "8.0", "8.9", "9.0a"] @@ -143,7 +146,9 @@ if cuda_ver is not None: if (major, minor) >= (13, 0): arches.append("10.0a") arches.append("10.3a") - arches.append("11.0a") + machine = os.environ.get("ARCH") or platform.machine() + if machine in ("aarch64", "arm64"): + arches.append("11.0a") arches.append("12.0f") elif (major, minor) >= (12, 9): arches.append("10.0a") From fa19a77ffb6aeb301cd66ffd7720ad2f34a9191c Mon Sep 17 00:00:00 2001 From: dierksen <1507580+dierksen@users.noreply.github.com> Date: Wed, 20 May 2026 16:57:09 -0500 Subject: [PATCH 8/8] Separate SM110 from SM10x --- .github/workflows/nightly-release.yml | 16 ++++++++++++++-- .github/workflows/pr-test.yml | 20 +++++++++++++++++--- .github/workflows/release.yml | 16 ++++++++++++++-- README.md | 2 +- build_utils.py | 7 +++++-- docs/installation.rst | 9 ++++++--- flashinfer/__main__.py | 5 +++-- flashinfer/aot.py | 21 ++++++++++++++------- 8 files changed, 74 insertions(+), 22 deletions(-) diff --git a/.github/workflows/nightly-release.yml b/.github/workflows/nightly-release.yml index b73c323dc1d..65784dea855 100644 --- a/.github/workflows/nightly-release.yml +++ b/.github/workflows/nightly-release.yml @@ -126,7 +126,15 @@ jobs: matrix: cuda: ["12.8", "12.9", "13.0"] arch: ['x86_64', 'aarch64'] - sm_family: ['sm9x', 'sm10x', 'sm12x'] + sm_family: ['sm9x', 'sm10x', 'sm110', 'sm12x'] + exclude: + # Thor (sm110) JIT-cache wheels are produced only for aarch64 CUDA 13.0. + - arch: x86_64 + sm_family: sm110 + - cuda: "12.8" + sm_family: sm110 + - cuda: "12.9" + sm_family: sm110 runs-on: [self-hosted, linux, "${{ matrix.arch == 'aarch64' && 'arm64' || 'x64' }}", cpu, on-demand] @@ -267,7 +275,11 @@ jobs: for cuda in 128 129 130; do for arch in x86_64 aarch64; do - for sm_family in sm9x sm10x sm12x; do + sm_families="sm9x sm10x sm12x" + if [ "$cuda" = "130" ] && [ "$arch" = "aarch64" ]; then + sm_families="sm9x sm10x sm110 sm12x" + fi + for sm_family in $sm_families; do ARTIFACT_NAME="jit-cache-cu${cuda}-${arch}-${sm_family}" echo "Processing ${ARTIFACT_NAME}..." diff --git a/.github/workflows/pr-test.yml b/.github/workflows/pr-test.yml index dc75d08c9e3..fbd46998e02 100644 --- a/.github/workflows/pr-test.yml +++ b/.github/workflows/pr-test.yml @@ -202,14 +202,23 @@ jobs: matrix: arch: [x64, arm64] cuda: [cu126, cu128, cu129, cu130] - sm_family: [sm9x, sm10x, sm12x] + sm_family: [sm9x, sm10x, sm110, sm12x] # cu126 only has sm9x archs (7.5 / 8.0 / 8.9 / 9.0a), so the - # sm10x and sm12x family builds would have empty arch lists. + # sm10x, sm110, and sm12x family builds would have empty arch lists. + # sm110 is only built for aarch64 CUDA 13.0 release coverage. exclude: - cuda: cu126 sm_family: sm10x + - cuda: cu126 + sm_family: sm110 - cuda: cu126 sm_family: sm12x + - cuda: cu128 + sm_family: sm110 + - cuda: cu129 + sm_family: sm110 + - arch: x64 + sm_family: sm110 env: DOCKER_IMAGE: flashinfer/flashinfer-ci-${{ matrix.cuda }}:${{ needs.setup.outputs.docker_tag }} steps: @@ -283,11 +292,16 @@ jobs: MATRIX='{"include":[' for arch in x64 arm64; do for cuda in cu126 cu128 cu129 cu130; do - for sm_family in sm9x sm10x sm12x; do + for sm_family in sm9x sm10x sm110 sm12x; do # Mirror the cu126 exclude rule from `aot-build-import`. if [ "$cuda" = "cu126" ] && [ "$sm_family" != "sm9x" ]; then continue fi + if [ "$sm_family" = "sm110" ]; then + if [ "$arch" != "arm64" ] || [ "$cuda" != "cu130" ]; then + continue + fi + fi MATRIX+='{"arch":"'$arch'","cuda":"'$cuda'","sm_family":"'$sm_family'"},' done done diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 35e980ec8c3..5cd8e41d6d5 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -177,7 +177,15 @@ jobs: matrix: cuda: ["12.8", "12.9", "13.0"] arch: ['x86_64', 'aarch64'] - sm_family: ['sm9x', 'sm10x', 'sm12x'] + sm_family: ['sm9x', 'sm10x', 'sm110', 'sm12x'] + exclude: + # Thor (sm110) JIT-cache wheels are produced only for aarch64 CUDA 13.0. + - arch: x86_64 + sm_family: sm110 + - cuda: "12.8" + sm_family: sm110 + - cuda: "12.9" + sm_family: sm110 runs-on: [self-hosted, linux, "${{ matrix.arch == 'aarch64' && 'arm64' || 'x64' }}", cpu, on-demand] @@ -350,7 +358,11 @@ jobs: for cuda in 128 129 130; do for arch in x86_64 aarch64; do - for sm_family in sm9x sm10x sm12x; do + sm_families="sm9x sm10x sm12x" + if [ "$cuda" = "130" ] && [ "$arch" = "aarch64" ]; then + sm_families="sm9x sm10x sm110 sm12x" + fi + for sm_family in $sm_families; do ARTIFACT_NAME="jit-cache-cu${cuda}-${arch}-${sm_family}" echo "Processing ${ARTIFACT_NAME}..." diff --git a/README.md b/README.md index 60da2c8b89f..f80d07e7667 100644 --- a/README.md +++ b/README.md @@ -107,7 +107,7 @@ pip install flashinfer-python flashinfer-cubin flashinfer install-jit-cache-wheel ``` -`flashinfer-jit-cache` is published as separate per-(CUDA, SM family) wheels — `sm9x` (Ampere/Ada/Hopper, ≤sm90), `sm10x` (Datacenter Blackwell, sm100/103/110), and `sm12x` (Consumer Blackwell, sm120/121) — because a single multi-arch wheel exceeds GitHub Releases' 2 GiB asset limit. The CLI checks all visible GPUs and resolves the right one for you when one wheel can cover the system. Blackwell-family wheels also retain the `sm80` base arch alongside their native Blackwell archs. +`flashinfer-jit-cache` is published as separate per-(CUDA, SM family) wheels — `sm9x` (Ampere/Ada/Hopper, ≤sm90), `sm10x` (Datacenter Blackwell, sm100/103), `sm110` (Thor), and `sm12x` (Consumer Blackwell, sm120/121) — because a single multi-arch wheel exceeds GitHub Releases' 2 GiB asset limit. The CLI checks all visible GPUs and resolves the right one for you when one wheel can cover the system. Blackwell and Thor wheels also retain the `sm80` base arch alongside their native archs. **For Blackwell (SM100+) CuTe DSL kernels**, install with the CUDA 13 extra to enable Blackwell-optimized kernels: diff --git a/build_utils.py b/build_utils.py index 660454d75a1..02666304333 100644 --- a/build_utils.py +++ b/build_utils.py @@ -21,9 +21,10 @@ from typing import Optional -SM_FAMILY_ORDER = ("sm9x", "sm10x", "sm12x") +SM_FAMILY_ORDER = ("sm9x", "sm10x", "sm110", "sm12x") SM_FAMILY_BASE_ARCHS = { "sm10x": ("8.0",), + "sm110": ("8.0",), "sm12x": ("8.0",), } @@ -56,8 +57,10 @@ def get_git_version(cwd: Optional[Path] = None) -> str: def sm_family_for_capability(major: int, minor: int) -> str: if major < 10: return "sm9x" - if major < 12: + if major == 10: return "sm10x" + if major == 11: + return "sm110" return "sm12x" diff --git a/docs/installation.rst b/docs/installation.rst index 5863d9bfb29..bac9759425d 100644 --- a/docs/installation.rst +++ b/docs/installation.rst @@ -55,15 +55,18 @@ manually, override the autodetection: .. code-block:: bash - # Datacenter Blackwell (sm100/103/110), CUDA 13.0 + # Datacenter Blackwell (sm100/103), CUDA 13.0 flashinfer install-jit-cache-wheel --cuda-version 13.0 --sm-family sm10x + # Thor (sm110), CUDA 13.0, aarch64 release wheel + flashinfer install-jit-cache-wheel --cuda-version 13.0 --sm-family sm110 + # Show the pip command without executing flashinfer install-jit-cache-wheel --dry-run The SM families are: ``sm9x`` (Ampere/Ada/Hopper, ≤sm90), ``sm10x`` (Datacenter Blackwell, -sm100/103/110), and ``sm12x`` (Consumer Blackwell, sm120/121). Blackwell-family -wheels also retain the ``sm80`` base arch alongside their native Blackwell archs. +sm100/103), ``sm110`` (Thor), and ``sm12x`` (Consumer Blackwell, sm120/121). Blackwell +and Thor wheels also retain the ``sm80`` base arch alongside their native archs. .. _install-from-source: diff --git a/flashinfer/__main__.py b/flashinfer/__main__.py index 85501d59b4f..15a74e3b615 100644 --- a/flashinfer/__main__.py +++ b/flashinfer/__main__.py @@ -88,7 +88,7 @@ def _detect_sm_family() -> str: if not torch.cuda.is_available() or torch.cuda.device_count() == 0: raise click.ClickException( "No CUDA device visible — cannot autodetect SM family. " - "Pass --sm-family {sm9x|sm10x|sm12x} or run on a host with the target GPU." + f"Pass --sm-family {{{'|'.join(SM_FAMILY_ORDER)}}} or run on a host with the target GPU." ) capabilities = [ @@ -376,7 +376,8 @@ def _build_package_install_command( type=click.Choice(SM_FAMILY_ORDER), help=( "Override SM family detection. sm9x = Ampere/Ada/Hopper (≤sm90), " - "sm10x = Datacenter Blackwell (sm100/103/110), sm12x = Consumer Blackwell (sm120/121)." + "sm10x = Datacenter Blackwell (sm100/103), sm110 = Thor, " + "sm12x = Consumer Blackwell (sm120/121)." ), ) @click.option( diff --git a/flashinfer/aot.py b/flashinfer/aot.py index db268ba8129..4a41afc1775 100644 --- a/flashinfer/aot.py +++ b/flashinfer/aot.py @@ -215,6 +215,7 @@ def gen_attention( use_logits_soft_cap_: List[bool], has_sm90: bool, has_sm100: bool, + has_sm110: bool, add_gemma: bool, add_oai_oss: bool, ) -> Iterator[JitSpec]: @@ -341,7 +342,7 @@ def gen_attention( # fmha_cutlass_sm100a # NOTE: currently there's only one uri. - if has_sm100: + if has_sm100 or has_sm110: yield gen_fmha_cutlass_sm100a_module( dtype_q=torch.bfloat16, dtype_kv=torch.bfloat16, @@ -354,6 +355,7 @@ def gen_attention( use_logits_soft_cap=False, ) + if has_sm100: # trtllm_gen_fmha yield gen_trtllm_gen_fmha_module() @@ -373,8 +375,8 @@ def gen_attention( use_profiler=False, ) - # MLA SM100 - if has_sm100: + # MLA SM100/SM110 + if has_sm100 or has_sm110: yield gen_mla_module() @@ -468,6 +470,7 @@ def gen_all_modules( has_sm120 = sm_capabilities.get("sm120", False) has_sm120f = sm_capabilities.get("sm120f", False) has_sm121 = sm_capabilities.get("sm121", False) + has_sm100_or_sm110 = has_sm100 or has_sm110 jit_specs += list( gen_attention( @@ -479,6 +482,7 @@ def gen_all_modules( use_logits_soft_cap_, has_sm90, has_sm100, + has_sm110, add_gemma, add_oai_oss, ) @@ -498,11 +502,14 @@ def gen_all_modules( jit_specs.append(gen_cutlass_fused_moe_sm90_module()) if has_sm100: jit_specs.append(gen_fp4_quantization_sm100_module()) + if has_sm100_or_sm110: jit_specs.append(gen_cutlass_fused_moe_sm100_module()) jit_specs.append(gen_gemm_sm100_module()) jit_specs.append(gen_gemm_sm100_module_cutlass_fp4()) jit_specs.append(gen_gemm_sm100_module_cutlass_fp8()) jit_specs.append(gen_gemm_sm100_module_cutlass_mxfp8()) + jit_specs.append(gen_mxfp8_quantization_sm100_module()) + if has_sm100: # Add TGV GEMM modules for both bf16 and fp16 jit_specs.append( gen_tgv_gemm_sm10x_module(torch.bfloat16, use_sm_100f=False) @@ -510,7 +517,6 @@ def gen_all_modules( jit_specs.append( gen_tgv_gemm_sm10x_module(torch.float16, use_sm_100f=False) ) - jit_specs.append(gen_mxfp8_quantization_sm100_module()) jit_specs.append(gen_trtllm_gen_gemm_module()) jit_specs.append(gen_trtllm_low_latency_gemm_module()) jit_specs.append(gen_trtllm_gen_fused_moe_sm100_module()) @@ -521,7 +527,7 @@ def gen_all_modules( ) jit_specs.append(gen_tgv_gemm_sm10x_module(torch.float16, use_sm_100f=True)) jit_specs.append(gen_moe_utils_module()) - if has_sm100 or has_sm103: + if has_sm100 or has_sm103 or has_sm110: jit_specs.append(gen_mm_bf16_cublaslt_module()) if has_sm103: jit_specs.append(gen_fp4_quantization_sm103_module()) @@ -558,9 +564,10 @@ def gen_all_modules( jit_specs.append(gen_trtllm_comm_module()) jit_specs.append(gen_trtllm_mnnvl_comm_module()) jit_specs.append(gen_moe_alltoall_module()) + if has_sm100_or_sm110: # dcp_alltoall: kernel itself supports SM90+, but ptxas 12.6.0 has # a known state-space inference bug on cp.async.bulk that aborts - # compilation. has_sm100 implies CUDA >= 12.8, which avoids the bug. + # compilation. has_sm100/has_sm110 imply CUDA >= 12.8, which avoids the bug. # SM90/SM12x users still get this via JIT. jit_specs.append(gen_dcp_alltoall_module()) jit_specs.append(gen_vllm_comm_module()) @@ -576,7 +583,7 @@ def gen_all_modules( gen_topk_module(), ] # Fused RMSNorm+SiLU: pre-compile all LUT configs (SM100+ only) - if has_sm100: + if has_sm100_or_sm110: for C in _SUPPORTED_C: for tokens in _SUPPORTED_TOKENS: for dtype in ["bf16", "fp8", "nvfp4"]: