Repository navigation
feat: add specialized gemm kernel for sm121 #3283
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. Weβll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Changes from 2 commits
01dac0a
e98dfcf
d57e017
c1505f0
f20efb1
11ad5e4
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,363 @@ | ||
| #!/usr/bin/env python3 | ||
| from __future__ import annotations | ||
|
|
||
| import argparse | ||
| import csv | ||
| import json | ||
| import math | ||
| import os | ||
| import statistics | ||
| from contextlib import contextmanager | ||
| from pathlib import Path | ||
| from typing import Any | ||
|
|
||
| import torch | ||
| import torch.nn.functional as F | ||
|
|
||
| import flashinfer | ||
| from flashinfer.env import ( | ||
| FLASHINFER_SPECIALIZED_KERNEL_DISABLE, | ||
| reset_specialized_kernel_env_cache, | ||
| ) | ||
| from flashinfer.gemm.specialized_kernels import ( | ||
| is_bmm_fp8_sm121_specialized_problem, | ||
| is_mm_fp4_sm121_specialized_problem, | ||
| ) | ||
| from flashinfer.testing.utils import bench_gpu_time | ||
|
|
||
|
|
||
| REPO_ROOT = Path(__file__).resolve().parents[1] | ||
| SPECIALIZED_KERNEL_DIR = REPO_ROOT / "flashinfer" / "gemm" / "specialized_kernels" | ||
| DEFAULT_WORKLOADS = { | ||
| "mm_fp4": SPECIALIZED_KERNEL_DIR / "mm_fp4_sm121" / "workloads.json", | ||
| "bmm_fp8": SPECIALIZED_KERNEL_DIR / "bmm_fp8_sm121" / "workloads.json", | ||
| } | ||
| THRESHOLDS = { | ||
| "mm_fp4": 0.97, | ||
| "bmm_fp8": 0.99, | ||
| } | ||
|
|
||
|
|
||
| @contextmanager | ||
| def specialized_routing_disabled(disabled: bool): | ||
| old_values = { | ||
| FLASHINFER_SPECIALIZED_KERNEL_DISABLE: os.environ.get( | ||
| FLASHINFER_SPECIALIZED_KERNEL_DISABLE | ||
| ), | ||
| } | ||
| value = "True" if disabled else "False" | ||
| os.environ[FLASHINFER_SPECIALIZED_KERNEL_DISABLE] = value | ||
| reset_specialized_kernel_env_cache() | ||
| try: | ||
| yield | ||
| finally: | ||
| for name, old_value in old_values.items(): | ||
| if old_value is None: | ||
| os.environ.pop(name, None) | ||
| else: | ||
| os.environ[name] = old_value | ||
| reset_specialized_kernel_env_cache() | ||
|
|
||
|
|
||
| @contextmanager | ||
| def autotune_mode(enabled: bool, cache: Path | None): | ||
| if enabled or cache is not None: | ||
| cache_path = str(cache) if cache is not None else None | ||
| with flashinfer.autotune(enabled, cache=cache_path): | ||
| yield | ||
| else: | ||
| yield | ||
|
|
||
|
|
||
| def load_workloads( | ||
| functions: list[str], m_filter: set[int] | None, limit: int | None | ||
| ) -> list[tuple[str, dict[str, Any]]]: | ||
| cases = [] | ||
| for function in functions: | ||
| workloads = json.loads(DEFAULT_WORKLOADS[function].read_text()) | ||
| for item in workloads: | ||
| if m_filter is not None and int(item["m"]) not in m_filter: | ||
| continue | ||
| cases.append((function, item)) | ||
| if limit is not None and len(cases) >= limit: | ||
| return cases | ||
| return cases | ||
|
|
||
|
|
||
| def to_float8(x: torch.Tensor, dtype: torch.dtype = torch.float8_e4m3fn): | ||
| finfo = torch.finfo(dtype) | ||
| min_val, max_val = x.aminmax() | ||
| amax = torch.maximum(min_val.abs(), max_val.abs()).clamp(min=1e-12) | ||
| scale = finfo.max / amax | ||
| x_scl_sat = (x * scale).clamp(min=finfo.min, max=finfo.max) | ||
| return x_scl_sat.to(dtype), scale.float().reciprocal() | ||
|
|
||
|
|
||
| def cosine(lhs: torch.Tensor, rhs: torch.Tensor) -> float: | ||
| return float( | ||
| F.cosine_similarity(lhs.reshape(-1).float(), rhs.reshape(-1).float(), dim=0) | ||
| ) | ||
|
|
||
|
|
||
| def make_mm_fp4_case(item: dict[str, Any], device: torch.device): | ||
| m, k, n = int(item["m"]), int(item["k"]), int(item["n"]) | ||
| a_ref = torch.randn((m, k), dtype=torch.bfloat16, device=device) | ||
| b_ref = torch.randn((n, k), dtype=torch.bfloat16, device=device) | ||
| reference = torch.mm(a_ref, b_ref.T) | ||
|
|
||
| a_global_sf = (448 * 6) / a_ref.float().abs().nan_to_num().max() | ||
| b_global_sf = (448 * 6) / b_ref.float().abs().nan_to_num().max() | ||
| a_fp4, a_sf = flashinfer.nvfp4_quantize( | ||
| a_ref, | ||
| a_global_sf, | ||
| sfLayout=flashinfer.SfLayout.layout_128x4, | ||
| do_shuffle=False, | ||
| ) | ||
| b_fp4, b_sf = flashinfer.nvfp4_quantize( | ||
| b_ref, | ||
| b_global_sf, | ||
| sfLayout=flashinfer.SfLayout.layout_128x4, | ||
| do_shuffle=False, | ||
| ) | ||
| alpha = 1.0 / (a_global_sf * b_global_sf) | ||
| out = torch.empty((m, n), dtype=torch.bfloat16, device=device) | ||
|
|
||
| def run(out_tensor: torch.Tensor): | ||
| return flashinfer.gemm.mm_fp4( | ||
| a_fp4, | ||
| b_fp4.T, | ||
| a_sf, | ||
| b_sf.T, | ||
| alpha, | ||
| torch.bfloat16, | ||
| out_tensor, | ||
| block_size=16, | ||
| use_8x4_sf_layout=False, | ||
| backend="b12x", | ||
| use_nvfp4=True, | ||
| ) | ||
|
|
||
| def is_routed(out_tensor: torch.Tensor): | ||
| return is_mm_fp4_sm121_specialized_problem( | ||
| a_fp4, | ||
| b_fp4.T, | ||
| a_sf, | ||
| b_sf.T, | ||
| alpha, | ||
| torch.bfloat16, | ||
| out_tensor, | ||
| block_size=16, | ||
| use_8x4_sf_layout=False, | ||
| backend="b12x", | ||
| use_nvfp4=True, | ||
| ) | ||
|
|
||
| return run, is_routed, out, reference | ||
|
|
||
|
|
||
| def make_bmm_fp8_case(item: dict[str, Any], device: torch.device): | ||
| b, m, k, n = int(item["b"]), int(item["m"]), int(item["k"]), int(item["n"]) | ||
| inp = torch.randn((b, m, k), dtype=torch.bfloat16, device=device) | ||
| inp_fp8, inp_inv_s = to_float8(inp) | ||
| mat2 = torch.randn((b, n, k), dtype=torch.bfloat16, device=device).transpose(-2, -1) | ||
| mat2_fp8, mat2_inv_s = to_float8(mat2) | ||
| reference = torch.bmm(inp, mat2) | ||
| out = torch.empty((b, m, n), dtype=torch.bfloat16, device=device) | ||
|
|
||
| def run(out_tensor: torch.Tensor): | ||
| return flashinfer.gemm.bmm_fp8( | ||
| inp_fp8, | ||
| mat2_fp8, | ||
| inp_inv_s, | ||
| mat2_inv_s, | ||
| torch.bfloat16, | ||
| out_tensor, | ||
| backend="cublas", | ||
| ) | ||
|
|
||
| def is_routed(out_tensor: torch.Tensor): | ||
| return is_bmm_fp8_sm121_specialized_problem( | ||
| inp_fp8, | ||
| mat2_fp8, | ||
| inp_inv_s, | ||
| mat2_inv_s, | ||
| torch.bfloat16, | ||
| out_tensor, | ||
| backend="cublas", | ||
| ) | ||
|
|
||
| return run, is_routed, out, reference | ||
|
|
||
|
|
||
| CASE_BUILDERS = { | ||
| "mm_fp4": make_mm_fp4_case, | ||
| "bmm_fp8": make_bmm_fp8_case, | ||
| } | ||
|
|
||
|
|
||
| def benchmark_one(function: str, item: dict[str, Any], args, device: torch.device): | ||
| run, is_routed, out, reference = CASE_BUILDERS[function](item, device) | ||
| disabled_out = torch.empty_like(out) | ||
| enabled_out = torch.empty_like(out) | ||
|
|
||
| def run_and_time(disabled: bool, target: torch.Tensor): | ||
| with specialized_routing_disabled(disabled): | ||
| if args.autotune: | ||
| with autotune_mode(True, args.autotune_cache): | ||
| run(target) | ||
| torch.cuda.synchronize(device) | ||
|
|
||
| with autotune_mode(False, args.autotune_cache): | ||
| result = run(target) | ||
| torch.cuda.synchronize(device) | ||
| snapshot = result.detach().clone() | ||
| times = bench_gpu_time( | ||
| fn=lambda: run(target), | ||
| dry_run_iters=args.dry_run_iters, | ||
| repeat_iters=args.num_iters, | ||
| enable_cupti=not args.no_cupti, | ||
| use_cuda_graph=not args.no_cuda_graph, | ||
| ) | ||
| return snapshot, float(statistics.median(times)) | ||
|
|
||
| disabled_snapshot, disabled_ms = run_and_time(True, disabled_out) | ||
| with specialized_routing_disabled(False): | ||
| route_active = is_routed(enabled_out) | ||
| enabled_snapshot, enabled_ms = run_and_time(False, enabled_out) | ||
| threshold = THRESHOLDS[function] | ||
| disabled_cosine = cosine(reference, disabled_snapshot) | ||
| enabled_cosine = cosine(reference, enabled_snapshot) | ||
| routed_cosine = cosine(disabled_snapshot, enabled_snapshot) | ||
| speedup = disabled_ms / enabled_ms | ||
|
|
||
| return { | ||
| "function": function, | ||
| "impl": item["impl"], | ||
| "autotune": args.autotune, | ||
| "autotune_cache": str(args.autotune_cache or ""), | ||
| "b": item.get("b", ""), | ||
| "m": item["m"], | ||
| "n": item["n"], | ||
| "k": item["k"], | ||
| "block_size": item.get("block_size", ""), | ||
| "threshold": threshold, | ||
| "disabled_ms": disabled_ms, | ||
| "enabled_ms": enabled_ms, | ||
| "route_active": route_active, | ||
| "speedup": speedup, | ||
| "improvement_pct": (speedup - 1.0) * 100.0, | ||
| "disabled_cosine": disabled_cosine, | ||
| "enabled_cosine": enabled_cosine, | ||
| "disabled_correct": disabled_cosine > threshold, | ||
| "enabled_correct": enabled_cosine > threshold, | ||
| "disabled_enabled_cosine": routed_cosine, | ||
| } | ||
|
|
||
|
|
||
| def geomean(values: list[float]) -> float: | ||
| return math.exp(sum(math.log(v) for v in values) / len(values)) | ||
|
Comment on lines
+257
to
+258
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. The
Please consider adding checks for these cases to improve the script's robustness. |
||
|
|
||
|
|
||
| def parse_args(): | ||
| parser = argparse.ArgumentParser() | ||
| parser.add_argument( | ||
| "--function", | ||
| choices=tuple(DEFAULT_WORKLOADS), | ||
| action="append", | ||
| help="Function to benchmark. Defaults to all specialized workloads.", | ||
| ) | ||
| parser.add_argument( | ||
| "--m", type=int, action="append", help="Only run the given M value." | ||
| ) | ||
| parser.add_argument("--limit", type=int, default=None) | ||
| parser.add_argument("--device", default="cuda") | ||
| parser.add_argument("--num-iters", type=int, default=30) | ||
| parser.add_argument("--dry-run-iters", type=int, default=5) | ||
| parser.add_argument( | ||
| "--autotune", | ||
| action="store_true", | ||
| help=( | ||
| "Profile FlashInfer autotune before each timing pass. Timed CUDA " | ||
| "graph capture uses the tuned cache without running profiling inside " | ||
| "capture." | ||
| ), | ||
| ) | ||
| parser.add_argument( | ||
| "--autotune-cache", | ||
| type=Path, | ||
| default=None, | ||
| help=( | ||
| "Optional autotuner config cache. With --autotune, loads then saves " | ||
| "configs; without --autotune, loads configs without profiling." | ||
| ), | ||
| ) | ||
| parser.add_argument( | ||
| "--no-cuda-graph", | ||
| action="store_true", | ||
| help="Disable CUDA graph timing. CUDA graph timing is enabled by default.", | ||
| ) | ||
| parser.add_argument( | ||
| "--no-cupti", | ||
| action="store_true", | ||
| help="Disable CUPTI timing. CUPTI timing is enabled by default.", | ||
| ) | ||
| parser.add_argument( | ||
| "--output-csv", | ||
| type=Path, | ||
| default=Path("specialized_gemm_routing_benchmark.csv"), | ||
| ) | ||
| parser.add_argument("--seed", type=int, default=42) | ||
| return parser.parse_args() | ||
|
|
||
|
|
||
| def main(): | ||
| args = parse_args() | ||
| if not torch.cuda.is_available(): | ||
| raise RuntimeError("This benchmark requires CUDA.") | ||
| device = torch.device(args.device) | ||
| torch.manual_seed(args.seed) | ||
|
|
||
| functions = args.function or list(DEFAULT_WORKLOADS) | ||
| rows = [] | ||
| m_filter = set(args.m) if args.m is not None else None | ||
| for function, item in load_workloads(functions, m_filter, args.limit): | ||
| result = benchmark_one(function, item, args, device) | ||
| rows.append(result) | ||
| print( | ||
| f"{function:14s} impl={result['impl']:8s} " | ||
| f"b={result['b']} m={result['m']} n={result['n']} k={result['k']} " | ||
| f"disabled={result['disabled_ms']:.6f} ms " | ||
| f"enabled={result['enabled_ms']:.6f} ms " | ||
| f"speedup={result['speedup']:.3f}x " | ||
| f"route_active={result['route_active']} " | ||
| f"correct={result['enabled_correct']}" | ||
| ) | ||
|
|
||
| if not rows: | ||
| raise RuntimeError("No workloads selected.") | ||
|
|
||
| fieldnames = list(rows[0].keys()) | ||
| with args.output_csv.open("w", newline="") as f: | ||
| writer = csv.DictWriter(f, fieldnames=fieldnames) | ||
| writer.writeheader() | ||
| writer.writerows(rows) | ||
|
|
||
| print("\nGeomean speedup, correct enabled cases only:") | ||
| for function in functions: | ||
| for impl in sorted({r["impl"] for r in rows if r["function"] == function}): | ||
| speeds = [ | ||
| r["speedup"] | ||
| for r in rows | ||
| if r["function"] == function | ||
| and r["impl"] == impl | ||
| and r["enabled_correct"] | ||
| and r["route_active"] | ||
| ] | ||
| if speeds: | ||
| print( | ||
| f"{function:14s} {impl:8s} {len(speeds):4d} cases {geomean(speeds):.4f}x" | ||
| ) | ||
|
|
||
|
|
||
| if __name__ == "__main__": | ||
| main() | ||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -83,6 +83,9 @@ | |
| gen_trtllm_gen_gemm_module, | ||
| gen_trtllm_low_latency_gemm_module, | ||
| ) | ||
| from .gemm.specialized_kernels import ( | ||
| gen_bmm_fp8_sm121_specialized_cuda_module, | ||
| ) | ||
| from .jit.mamba import ( | ||
| gen_selective_state_update_module, | ||
| gen_selective_state_update_sm90_module, | ||
|
|
@@ -543,6 +546,9 @@ def gen_all_modules( | |
| if has_sm120f: | ||
| jit_specs.append(gen_fp4_quantization_sm120f_module()) | ||
|
|
||
| if has_sm121: | ||
| jit_specs.append(gen_bmm_fp8_sm121_specialized_cuda_module()) | ||
|
Comment on lines
+551
to
+552
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. π§© Analysis chainπ Script executed: #!/bin/bash
# Check if bmm_fp8_sm121 module supports SM120 or is SM121-only
echo "=== Checking supported_compute_capability in bmm_fp8_sm121 module ==="
rg -n "supported_compute_capability" flashinfer/gemm/specialized_kernels/bmm_fp8_sm121/
echo ""
echo "=== Checking for SM120/SM121 version checks in bmm_fp8_sm121 module ==="
rg -n -C3 "\b(120|121)\b" flashinfer/gemm/specialized_kernels/bmm_fp8_sm121/ --glob '*.py'
echo ""
echo "=== Checking if routing logic distinguishes SM120 from SM121 ==="
rg -n -C3 "is_bmm_fp8_sm121_specialized_problem" flashinfer/gemm/specialized_kernels/Repository: flashinfer-ai/flashinfer Length of output: 3789 π Script executed: #!/bin/bash
# Get context around lines 551-552 in aot.py
echo "=== Context around lines 540-560 in aot.py ==="
sed -n '540,560p' flashinfer/aot.py
echo ""
echo "=== Check comments around SM120/SM121 handling ==="
sed -n '495,560p' flashinfer/aot.py | head -70Repository: flashinfer-ai/flashinfer Length of output: 4368 π Script executed: #!/bin/bash
# Examine the routing function implementation
echo "=== Implementation of is_bmm_fp8_sm121_specialized_problem ==="
rg -n -A30 "^def is_bmm_fp8_sm121_specialized_problem" flashinfer/gemm/specialized_kernels/bmm_fp8_sm121/bmm_fp8_sm121.py
echo ""
echo "=== Check if SM121/SM120 checking logic exists ==="
rg -n -i "sm120\|sm121\|compute.capability\|cuda_capability" flashinfer/gemm/specialized_kernels/bmm_fp8_sm121/Repository: flashinfer-ai/flashinfer Length of output: 1109 π Script executed: #!/bin/bash
# Check how other SM121 specialized modules are registered
echo "=== How mm_fp4_sm121 is registered ==="
rg -n -B5 -A5 "gen_mm_fp4_sm121_specialized_cuda_module" flashinfer/aot.py | head -30Repository: flashinfer-ai/flashinfer Length of output: 107 π Script executed: #!/bin/bash
# Find sm121a_nvcc_flags definition
echo "=== Find sm121a_nvcc_flags definition ==="
rg -n "sm121a_nvcc_flags" flashinfer/
echo ""
echo "=== Check if there are SM120-specific flags ==="
rg -n "sm120.*nvcc_flags" flashinfer/
echo ""
echo "=== Look at the gen_jit_spec function to understand compilation ==="
rg -n -A10 "^def gen_jit_spec" flashinfer/Repository: flashinfer-ai/flashinfer Length of output: 2870 π Script executed: #!/bin/bash
# Check what flags are being used for SM120 modules vs SM121
echo "=== Check how other SM120 modules compile (like gen_gemm_sm120_module) ==="
rg -n -B5 -A15 "def gen_gemm_sm120_module\(\)" flashinfer/ | head -50
echo ""
echo "=== Look for nvcc_flags usage pattern ==="
rg -n "nvcc_flags" flashinfer/ | grep -E "(sm12|sm121|sm120)"Repository: flashinfer-ai/flashinfer Length of output: 3513 π Script executed: #!/bin/bash
# Verify that sm121a_nvcc_flags truly targets SM121 only
echo "=== Definition of sm121a_nvcc_flags ==="
rg -n -A2 "^sm121a_nvcc_flags" flashinfer/jit/core.py
echo ""
echo "=== Confirm SM120 uses different flags ==="
rg -n -A2 "^sm120a_nvcc_flags" flashinfer/jit/core.py
echo ""
echo "=== Check if bmm_fp8_sm121 module should have SM120 variant ==="
ls -la flashinfer/gemm/specialized_kernels/ | grep bmm_fp8Repository: flashinfer-ai/flashinfer Length of output: 691 π Script executed: #!/bin/bash
# Check if there are any SM120-specific bmm_fp8 modules
echo "=== Search for any bmm_fp8 variants ==="
find flashinfer/gemm/specialized_kernels -type d -name "*bmm_fp8*"
echo ""
echo "=== Check the git history or comments about SM120 support for bmm_fp8 ==="
rg -n -B3 -A3 "bmm_fp8.*sm12" flashinfer/aot.pyRepository: flashinfer-ai/flashinfer Length of output: 699 Change condition to The π€ Prompt for AI Agents |
||
|
|
||
| if add_comm: | ||
| from .jit.comm import ( | ||
| gen_comm_alltoall_module, | ||
|
|
||
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
The calculation for
speedupcould result in aZeroDivisionErrorifenabled_msis zero. While this may be unlikely for GPU timings, it would be safer to handle this edge case to prevent the benchmark from crashing.