Skip to content
Closed
Show file tree
Hide file tree
Changes from 5 commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
50 changes: 29 additions & 21 deletions .github/workflows/nightly-release.yml
Original file line number Diff line number Diff line change
Expand Up @@ -126,6 +126,7 @@ jobs:
matrix:
cuda: ["12.8", "12.9", "13.0"]
arch: ['x86_64', 'aarch64']
sm_family: ['sm9x', 'sm10x', 'sm12x']

runs-on: [self-hosted, linux, "${{ matrix.arch == 'aarch64' && 'arm64' || 'x64' }}", cpu, on-demand]

Expand All @@ -144,6 +145,7 @@ jobs:
echo "RAM: $(free -h | awk '/^Mem:/ {print $7 " available out of " $2}')"
echo "Disk: $(df -h / | awk 'NR==2 {print $4 " available out of " $2}')"
echo "Architecture: $(uname -m)"
echo "SM family: ${{ matrix.sm_family }}"

- name: Checkout code
uses: actions/checkout@v4
Expand Down Expand Up @@ -176,6 +178,7 @@ jobs:
-e FLASHINFER_DEV_RELEASE_SUFFIX="${FLASHINFER_DEV_RELEASE_SUFFIX}" \
-e ARCH="${{ matrix.arch }}" \
-e FLASHINFER_CUDA_ARCH_LIST="${FLASHINFER_CUDA_ARCH_LIST}" \
-e FLASHINFER_JIT_CACHE_SM_FAMILY="${{ matrix.sm_family }}" \
-e FLASHINFER_NVCC_THREADS="${FLASHINFER_NVCC_THREADS:-}" \
-e SCCACHE_BUCKET="${{ vars.SCCACHE_BUCKET }}" \
-e AWS_ACCESS_KEY_ID="${{ secrets.SCCACHE_AWS_ACCESS_KEY_ID }}" \
Expand All @@ -193,7 +196,7 @@ jobs:
id: artifact-name
run: |
CUDA_NO_DOT=$(echo "${{ matrix.cuda }}" | tr -d '.')
echo "name=jit-cache-cu${CUDA_NO_DOT}-${{ matrix.arch }}" >> $GITHUB_OUTPUT
echo "name=jit-cache-cu${CUDA_NO_DOT}-${{ matrix.arch }}-${{ matrix.sm_family }}" >> $GITHUB_OUTPUT

- name: Upload flashinfer-jit-cache artifact
uses: actions/upload-artifact@v4
Expand Down Expand Up @@ -257,29 +260,32 @@ jobs:
env:
GH_TOKEN: ${{ github.token }}
run: |
# Upload jit-cache wheels one at a time to avoid OOM
# Each wheel can be several GB, so we download, upload, delete, repeat
# Upload jit-cache wheels one at a time to avoid OOM.
# Wheels are split per (CUDA, CPU-arch, SM family) to stay under
# GitHub Releases' 2 GiB per-asset limit.
mkdir -p dist-jit-cache

for cuda in 128 129 130; do
for arch in x86_64 aarch64; do
ARTIFACT_NAME="jit-cache-cu${cuda}-${arch}"
echo "Processing ${ARTIFACT_NAME}..."

# Download this specific artifact
gh run download ${{ github.run_id }} -n "${ARTIFACT_NAME}" -D dist-jit-cache/ || {
echo "Warning: Failed to download ${ARTIFACT_NAME}, skipping..."
continue
}

# Upload to release
if [ -n "$(ls -A dist-jit-cache/)" ]; then
gh release upload "${{ needs.setup.outputs.release_tag }}" dist-jit-cache/* --clobber
echo "✅ Uploaded ${ARTIFACT_NAME}"
fi

# Clean up to save disk space before next iteration
rm -rf dist-jit-cache/*
for sm_family in sm9x sm10x sm12x; do
ARTIFACT_NAME="jit-cache-cu${cuda}-${arch}-${sm_family}"
echo "Processing ${ARTIFACT_NAME}..."

# Download this specific artifact
gh run download ${{ github.run_id }} -n "${ARTIFACT_NAME}" -D dist-jit-cache/ || {
echo "Warning: Failed to download ${ARTIFACT_NAME}, skipping..."
continue
}

# Upload to release
if [ -n "$(ls -A dist-jit-cache/)" ]; then
gh release upload "${{ needs.setup.outputs.release_tag }}" dist-jit-cache/* --clobber
echo "✅ Uploaded ${ARTIFACT_NAME}"
fi

# Clean up to save disk space before next iteration
rm -rf dist-jit-cache/*
done
done
done

Expand Down Expand Up @@ -326,9 +332,11 @@ jobs:
path: dist-cubin/

- name: Download flashinfer-jit-cache artifact
# The test runner is an sm86 (Ampere) host, which falls under the
# sm9x family wheel. Update the family if test runners change.
uses: actions/download-artifact@v4
with:
name: jit-cache-cu${{ matrix.cuda == '12.9' && '129' || '130' }}-x86_64
name: jit-cache-cu${{ matrix.cuda == '12.9' && '129' || '130' }}-x86_64-sm9x
path: dist-jit-cache/

- name: Get Docker image tag
Expand Down
22 changes: 19 additions & 3 deletions .github/workflows/pr-test.yml
Original file line number Diff line number Diff line change
Expand Up @@ -184,7 +184,7 @@ jobs:
# AOT Build Import Tests (Spot + On-Demand Rerun)
# ---------------------------------------------------------------------------
aot-build-import:
name: AOT Build Import (${{ matrix.arch }}, ${{ matrix.cuda }})
name: AOT Build Import (${{ matrix.arch }}, ${{ matrix.cuda }}, ${{ matrix.sm_family }})
needs: [gate, setup]
if: |
needs.gate.outputs.authorized == 'true' &&
Expand All @@ -202,6 +202,14 @@ jobs:
matrix:
arch: [x64, arm64]
cuda: [cu126, cu128, cu129, cu130]
sm_family: [sm9x, sm10x, sm12x]
# cu126 only has sm9x archs (7.5 / 8.0 / 8.9 / 9.0a), so the
# sm10x and sm12x family builds would have empty arch lists.
exclude:
- cuda: cu126
sm_family: sm10x
- cuda: cu126
sm_family: sm12x
env:
DOCKER_IMAGE: flashinfer/flashinfer-ci-${{ matrix.cuda }}:${{ needs.setup.outputs.docker_tag }}
steps:
Expand Down Expand Up @@ -240,6 +248,7 @@ jobs:
AWS_SECRET_ACCESS_KEY: ${{ secrets.SCCACHE_AWS_SECRET_ACCESS_KEY }}
run: |
bash ci/bash.sh "${DOCKER_IMAGE}" \
-e FLASHINFER_JIT_CACHE_SM_FAMILY "${{ matrix.sm_family }}" \
-e SCCACHE_BUCKET "${SCCACHE_BUCKET}" \
-e SCCACHE_REGION "${SCCACHE_REGION}" \
-e AWS_ACCESS_KEY_ID \
Expand Down Expand Up @@ -274,14 +283,20 @@ jobs:
MATRIX='{"include":['
for arch in x64 arm64; do
for cuda in cu126 cu128 cu129 cu130; do
MATRIX+='{"arch":"'$arch'","cuda":"'$cuda'"},'
for sm_family in sm9x sm10x sm12x; do
# Mirror the cu126 exclude rule from `aot-build-import`.
if [ "$cuda" = "cu126" ] && [ "$sm_family" != "sm9x" ]; then
continue
fi
MATRIX+='{"arch":"'$arch'","cuda":"'$cuda'","sm_family":"'$sm_family'"},'
done
done
done
MATRIX="${MATRIX%,}]}"
echo "rerun_matrix=$MATRIX" >> $GITHUB_OUTPUT

aot-build-import-rerun:
name: AOT Build Import Rerun (${{ matrix.arch }}, ${{ matrix.cuda }})
name: AOT Build Import Rerun (${{ matrix.arch }}, ${{ matrix.cuda }}, ${{ matrix.sm_family }})
needs: [setup, analyze-aot-failure]
if: |
!cancelled() &&
Expand Down Expand Up @@ -332,6 +347,7 @@ jobs:
AWS_SECRET_ACCESS_KEY: ${{ secrets.SCCACHE_AWS_SECRET_ACCESS_KEY }}
run: |
bash ci/bash.sh "${DOCKER_IMAGE}" \
-e FLASHINFER_JIT_CACHE_SM_FAMILY "${{ matrix.sm_family }}" \
-e SCCACHE_BUCKET "${SCCACHE_BUCKET}" \
-e SCCACHE_REGION "${SCCACHE_REGION}" \
-e AWS_ACCESS_KEY_ID \
Expand Down
75 changes: 51 additions & 24 deletions .github/workflows/release.yml
Original file line number Diff line number Diff line change
Expand Up @@ -42,7 +42,12 @@ jobs:
- name: Checkout code
uses: actions/checkout@v4
with:
ref: ${{ github.event_name == 'pull_request' && github.head_ref || inputs.tag }}
# On PRs the head branch may live on a fork, so we must explicitly
# set `repository:` to the head's repo and pin to head.sha (stable
# across re-pushes during the run). On `workflow_dispatch` the
# default repository is correct and we check out the input tag.
ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || inputs.tag }}
repository: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name || github.repository }}
submodules: true

- name: Verify tag matches version.txt
Expand Down Expand Up @@ -90,7 +95,12 @@ jobs:
- name: Checkout code
uses: actions/checkout@v4
with:
ref: ${{ github.event_name == 'pull_request' && github.head_ref || inputs.tag }}
# On PRs the head branch may live on a fork, so we must explicitly
# set `repository:` to the head's repo and pin to head.sha (stable
# across re-pushes during the run). On `workflow_dispatch` the
# default repository is correct and we check out the input tag.
ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || inputs.tag }}
repository: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name || github.repository }}
submodules: true

- name: Set up Python
Expand Down Expand Up @@ -124,7 +134,12 @@ jobs:
- name: Checkout code
uses: actions/checkout@v4
with:
ref: ${{ github.event_name == 'pull_request' && github.head_ref || inputs.tag }}
# On PRs the head branch may live on a fork, so we must explicitly
# set `repository:` to the head's repo and pin to head.sha (stable
# across re-pushes during the run). On `workflow_dispatch` the
# default repository is correct and we check out the input tag.
ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || inputs.tag }}
repository: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name || github.repository }}
submodules: true

- name: Set up Python
Expand Down Expand Up @@ -162,6 +177,7 @@ jobs:
matrix:
cuda: ["12.8", "12.9", "13.0"]
arch: ['x86_64', 'aarch64']
sm_family: ['sm9x', 'sm10x', 'sm12x']

runs-on: [self-hosted, linux, "${{ matrix.arch == 'aarch64' && 'arm64' || 'x64' }}", cpu, on-demand]

Expand All @@ -172,11 +188,17 @@ jobs:
echo "RAM: $(free -h | awk '/^Mem:/ {print $7 " available out of " $2}')"
echo "Disk: $(df -h / | awk 'NR==2 {print $4 " available out of " $2}')"
echo "Architecture: $(uname -m)"
echo "SM family: ${{ matrix.sm_family }}"

- name: Checkout code
uses: actions/checkout@v4
with:
ref: ${{ github.event_name == 'pull_request' && github.head_ref || inputs.tag }}
# On PRs the head branch may live on a fork, so we must explicitly
# set `repository:` to the head's repo and pin to head.sha (stable
# across re-pushes during the run). On `workflow_dispatch` the
# default repository is correct and we check out the input tag.
ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || inputs.tag }}
repository: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name || github.repository }}
submodules: true

- name: Build wheel in container
Expand All @@ -203,6 +225,7 @@ jobs:
-e FLASHINFER_LOCAL_VERSION="$FLASHINFER_LOCAL_VERSION" \
-e ARCH="${{ matrix.arch }}" \
-e FLASHINFER_CUDA_ARCH_LIST="${FLASHINFER_CUDA_ARCH_LIST}" \
-e FLASHINFER_JIT_CACHE_SM_FAMILY="${{ matrix.sm_family }}" \
-e FLASHINFER_NVCC_THREADS="${FLASHINFER_NVCC_THREADS:-}" \
-e SCCACHE_BUCKET="${{ vars.SCCACHE_BUCKET }}" \
-e AWS_ACCESS_KEY_ID="${{ secrets.SCCACHE_AWS_ACCESS_KEY_ID }}" \
Expand All @@ -220,7 +243,7 @@ jobs:
id: artifact-name
run: |
CUDA_NO_DOT=$(echo "${{ matrix.cuda }}" | tr -d '.')
echo "name=jit-cache-cu${CUDA_NO_DOT}-${{ matrix.arch }}" >> $GITHUB_OUTPUT
echo "name=jit-cache-cu${CUDA_NO_DOT}-${{ matrix.arch }}-${{ matrix.sm_family }}" >> $GITHUB_OUTPUT

- name: Upload flashinfer-jit-cache artifact
uses: actions/upload-artifact@v4
Expand Down Expand Up @@ -319,29 +342,33 @@ jobs:
env:
GH_TOKEN: ${{ github.token }}
run: |
# Upload jit-cache wheels one at a time to avoid OOM
# Each wheel can be several GB, so we download, upload, delete, repeat
# Upload jit-cache wheels one at a time to avoid OOM.
# Each wheel can be ~1 GB, so we download, upload, delete, repeat.
# Wheels are split per (CUDA, CPU-arch, SM family) to stay under
# GitHub Releases' 2 GiB per-asset limit.
mkdir -p dist-jit-cache

for cuda in 128 129 130; do
for arch in x86_64 aarch64; do
ARTIFACT_NAME="jit-cache-cu${cuda}-${arch}"
echo "Processing ${ARTIFACT_NAME}..."

# Download this specific artifact
gh run download ${{ github.run_id }} -n "${ARTIFACT_NAME}" -D dist-jit-cache/ || {
echo "Warning: Failed to download ${ARTIFACT_NAME}, skipping..."
continue
}

# Upload to release
if [ -n "$(ls -A dist-jit-cache/)" ]; then
gh release upload "${{ needs.setup.outputs.release_tag }}" dist-jit-cache/* --clobber
echo "✅ Uploaded ${ARTIFACT_NAME}"
fi

# Clean up to save disk space before next iteration
rm -rf dist-jit-cache/*
for sm_family in sm9x sm10x sm12x; do
ARTIFACT_NAME="jit-cache-cu${cuda}-${arch}-${sm_family}"
echo "Processing ${ARTIFACT_NAME}..."

# Download this specific artifact
gh run download ${{ github.run_id }} -n "${ARTIFACT_NAME}" -D dist-jit-cache/ || {
echo "Warning: Failed to download ${ARTIFACT_NAME}, skipping..."
continue
}

# Upload to release
if [ -n "$(ls -A dist-jit-cache/)" ]; then
gh release upload "${{ needs.setup.outputs.release_tag }}" dist-jit-cache/* --clobber
echo "✅ Uploaded ${ARTIFACT_NAME}"
fi

# Clean up to save disk space before next iteration
rm -rf dist-jit-cache/*
done
done
done

Expand Down
17 changes: 13 additions & 4 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -102,10 +102,13 @@ pip install flashinfer-python

```bash
pip install flashinfer-python flashinfer-cubin
# JIT cache (replace cu129 with your CUDA version)
pip install flashinfer-jit-cache --index-url https://flashinfer.ai/whl/cu129
# JIT cache: autodetect CUDA + GPU SM family and run the matching pip install.
# Use --dry-run to preview, --sm-family / --cuda-version to override.
flashinfer install-jit-cache-wheel
Comment on lines +105 to +107

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Can you make sure to test this within regular python venvs and also with uv venvs? I commonly use uv pip install so I forget if there sometimes can be issues mixing pip install and uv pip install within the same environment

```

`flashinfer-jit-cache` is published as separate per-(CUDA, SM family) wheels — `sm9x` (Ampere/Ada/Hopper, ≤sm90), `sm10x` (Datacenter Blackwell, sm100/103/110), and `sm12x` (Consumer Blackwell, sm120/121) — because a single multi-arch wheel exceeds GitHub Releases' 2 GiB asset limit. The CLI resolves the right one for you.

**For Blackwell (SM100+) CuTe DSL kernels**, install with the CUDA 13 extra to enable Blackwell-optimized kernels:

```bash
Expand Down Expand Up @@ -168,6 +171,12 @@ export FLASHINFER_CUDA_ARCH_LIST="7.5 8.0 8.9 9.0a 10.0a 10.3a 11.0a 12.0f"
cd flashinfer-jit-cache
python -m build --no-isolation --wheel
python -m pip install dist/*.whl

# Or build a per-SM-family wheel matching a release artifact (smaller; one GPU family).
# The local-version on the resulting wheel will include the family (e.g. +cu130.sm10x).
export FLASHINFER_CUDA_ARCH_LIST="10.0a 10.3a 11.0a"
export FLASHINFER_JIT_CACHE_SM_FAMILY="sm10x"
python -m build --no-isolation --wheel
```

For more details, see the [Install from Source documentation](https://docs.flashinfer.ai/installation.html#install-from-source).
Expand All @@ -178,8 +187,8 @@ For more details, see the [Install from Source documentation](https://docs.flash
pip install -U --pre flashinfer-python --index-url https://flashinfer.ai/whl/nightly/ --no-deps
pip install flashinfer-python # Install dependencies from PyPI
pip install -U --pre flashinfer-cubin --index-url https://flashinfer.ai/whl/nightly/
# JIT cache (replace cu129 with your CUDA version)
pip install -U --pre flashinfer-jit-cache --index-url https://flashinfer.ai/whl/nightly/cu129
# JIT cache: autodetect CUDA + GPU and pull from the nightly index
flashinfer install-jit-cache-wheel --nightly
```

### CLI Tools
Expand Down
44 changes: 44 additions & 0 deletions build_utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,9 @@
from typing import Optional


SM_FAMILY_ORDER = ("sm9x", "sm10x", "sm12x")


def get_git_version(cwd: Optional[Path] = None) -> str:
"""
Get git commit hash.
Expand All @@ -44,3 +47,44 @@ def get_git_version(cwd: Optional[Path] = None) -> str:
return git_version
except Exception:
return "unknown"


def sm_family_for_capability(major: int, minor: int) -> str:
if major < 10:
return "sm9x"
if major < 12:
return "sm10x"
return "sm12x"


def parse_cuda_arch_entry(entry: str) -> tuple[int, int]:
"""Parse arch entries like '9.0a', '90', or '120f' into (major, minor)."""
entry = entry.strip()
if "." in entry:
major_str, minor_str = entry.split(".", 1)
major_digits = "".join(c for c in major_str if c.isdigit())
minor_digits = "".join(c for c in minor_str if c.isdigit())
if not major_digits:
raise ValueError(f"Invalid CUDA arch entry: {entry!r}")
return int(major_digits), int(minor_digits) if minor_digits else 0

digits = "".join(c for c in entry if c.isdigit())
if not digits:
raise ValueError(f"Invalid CUDA arch entry: {entry!r}")
if len(digits) >= 2:
return int(digits[:-1]), int(digits[-1])
return int(digits), 0


def filter_arch_list_for_sm_family(arch_list: str, family: str) -> str:
"""Filter a space-separated FLASHINFER_CUDA_ARCH_LIST for one SM family."""
if family not in SM_FAMILY_ORDER:
raise ValueError(
f"Invalid SM family {family!r}; expected one of {SM_FAMILY_ORDER}"
)
kept = []
for entry in arch_list.split():
major, minor = parse_cuda_arch_entry(entry)
if sm_family_for_capability(major, minor) == family:
kept.append(entry)
return " ".join(kept)
Loading
Loading