Skip to content
Merged
Show file tree
Hide file tree
Changes from 25 commits
Commits
Show all changes
31 commits
Select commit Hold shift + click to select a range
1c1e7b2
First commit
bkryu May 27, 2026
8082252
some cleanup and more scaffolding
bkryu May 27, 2026
0ee01fc
add_cudnn_w4a16
yanqinz2 May 29, 2026
c124a7d
Merge branch 'main' into mm_fp4_w4a16_cudnn_and_dsl
bkryu Jun 2, 2026
bec10ef
Add cute-dsl kernel
bkryu Jun 4, 2026
57f6b9c
Improve cute dsl kernel
bkryu Jun 4, 2026
8d90190
Relax N % 128
bkryu Jun 4, 2026
b7afa24
cute-dsl autotuning
bkryu Jun 4, 2026
3079dc5
Checkpoint
bkryu Jun 5, 2026
fc33c4e
Finalize kernel. Now need cleanup
bkryu Jun 8, 2026
2afdccd
Add missing file
bkryu Jun 8, 2026
ee22fbf
Temporarily add Marlin benchmark
bkryu Jun 8, 2026
8d00aa9
Cleanup torch backend
bkryu Jun 8, 2026
5d8ff5e
Merge branch 'main' into mm_fp4_w4a16_cudnn_and_dsl
bkryu Jun 8, 2026
8e0c566
Cleanup1
bkryu Jun 8, 2026
d2964c2
Cleanup2
bkryu Jun 9, 2026
823fb46
Merge branch 'main' into mm_fp4_w4a16_cudnn_and_dsl
bkryu Jun 11, 2026
95e1f0e
Remove temporary vLLM Marlin comparison benchmark
bkryu Jun 11, 2026
b7e2f03
Address review comments
bkryu Jun 12, 2026
c8c554d
Merge branch 'main' into mm_fp4_w4a16_cudnn_and_dsl
bkryu Jun 12, 2026
301eb4d
Fix CI failures
bkryu Jun 12, 2026
039e01f
Merge branch 'main' into mm_fp4_w4a16_cudnn_and_dsl
bkryu Jun 12, 2026
3025cf1
Merge mm_fp4_w4a16 into mm_fp4
bkryu Jun 15, 2026
d2cf941
Cleanup
bkryu Jun 15, 2026
5a20b73
Address review comments
bkryu Jun 15, 2026
b9ef7d4
Add autotuning tests to unit tests
bkryu Jun 15, 2026
a1070f5
Address review comments. Refactor into mm_bf16_fp4
bkryu Jun 16, 2026
6dc1e98
Cleanup
bkryu Jun 16, 2026
3e02472
Rename W4A16 to bf16_fp4
bkryu Jun 16, 2026
551882b
Move kernel file into cute_dsl directory
bkryu Jun 16, 2026
6a332bb
Split gemm_bf16_fp4.py
bkryu Jun 16, 2026
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 5 additions & 4 deletions benchmarks/flashinfer_benchmark.py
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,6 @@
from routines.flashinfer_benchmark_utils import (
benchmark_apis,
full_output_columns,
output_column_dict,
)


Expand Down Expand Up @@ -75,9 +74,11 @@ def run_test(args):
if args.output_path is not None:
with open(args.output_path, "a") as fout:
for cur_res in res:
for key in output_column_dict["general"]:
# Only set from args if the routine hasn't already set a value
# This preserves routine-specific formatting while providing defaults
for key in full_output_columns:
# Backfill every output column the routine didn't set: from
# args when available, else "". Covers columns belonging to
# other routines (e.g. attention's s_qo) that would otherwise
# KeyError below. Routine-set values are preserved.
if key not in cur_res or cur_res[key] == "":
cur_res[key] = getattr(args, key, "")

Expand Down
1 change: 1 addition & 0 deletions benchmarks/routines/flashinfer_benchmark_utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -199,6 +199,7 @@
"bmm_fp8",
"bmm_mxfp8",
"mm_fp4",
"mm_fp4_w4a16",
"mm_mxfp8",
"mm_bf16",
"bmm_bf16",
Expand Down
251 changes: 251 additions & 0 deletions benchmarks/routines/gemm.py
Original file line number Diff line number Diff line change
Expand Up @@ -44,6 +44,8 @@ def run_gemm_test(args):
return testBmmMxfp8(args)
elif args.routine == "mm_fp4":
return testMmFp4(args)
elif args.routine == "mm_fp4_w4a16":
return testMmFp4W4A16(args)
elif args.routine == "mm_mxfp8":
return testMmMxfp8(args)
elif args.routine == "mm_bf16":
Expand Down Expand Up @@ -200,6 +202,11 @@ def parse_gemm_args(line, parser):
args.input_dtype = "bfloat16"
if not has_mat2_dtype_arg:
args.mat2_dtype = "bfloat16"
if args.routine == "mm_fp4_w4a16":
if not has_backends_arg:
args.backends = ["cute-dsl"]
if not has_input_dtype_arg:
args.input_dtype = "bfloat16"
if args.verbose >= 1:
print(f"[INFO] {args = }")
return args
Expand Down Expand Up @@ -1326,6 +1333,250 @@ def run_backend(
return res


# E2M1 (FP4) codebook, signed (codes 0-7 positive, 8-15 negative), matching
# ``flashinfer.nvfp4_quantize``.
_E2M1_CODEBOOK_FP32 = (
0.0, 0.5, 1.0, 1.5, 2.0, 3.0, 4.0, 6.0,
-0.0, -0.5, -1.0, -1.5, -2.0, -3.0, -4.0, -6.0,
) # fmt: skip


def _dequantize_w4a16_fp4_ref(b, b_descale, alpha, n, k, block_size):
"""fp32 ground-truth dequant of the canonical nvfp4 weight (refcheck gold):
unpack FP4 nibbles -> codebook -> per-block SF (unswizzled) -> optional
alpha. Returns the ``(N, K)`` fp32 weight matrix."""
import torch
from flashinfer.gemm.gemm_w4a16 import _unswizzle_sf_128x4

device = b.device
k_sf = k // block_size
lut = torch.tensor(_E2M1_CODEBOOK_FP32, dtype=torch.float32, device=device)
b_int = b.to(torch.int64)
codes = torch.stack([b_int & 0xF, (b_int >> 4) & 0xF], dim=-1).reshape(n, k)
values = lut[codes]
sf = _unswizzle_sf_128x4(b_descale, n, k_sf).view(torch.float8_e4m3fn)
sf_expanded = sf.to(torch.float32).repeat_interleave(block_size, dim=1)
weight = values * sf_expanded
if alpha is not None:
weight = weight * alpha.to(torch.float32)
return weight


def testMmFp4W4A16(args):
"""Benchmark mm_fp4 in W4A16 weight-only mode (bf16 activation, FP4 weight).

Weights are produced by ``flashinfer.nvfp4_quantize(sfLayout=layout_128x4)``
-- the same canonical format the new API expects. Per-backend prep
(``prepare_w4a16_fp4_weights``) runs once at setup and is NOT timed;
only the ``mm_fp4`` (W4A16 mode) call is.

Constraints:
* N must be divisible by 64 (the weight-prepack / kernel N tile).
The 128x4 SF swizzle only pads N to 128; prepare_w4a16_fp4_weights
unswizzles the padded tail, so any N % 64 == 0 works.
* K must be divisible by 16 (FP4 block size).
* input_dtype must be bfloat16 (the only A dtype currently
supported; fp16 deferred).

Refcheck uses an fp32 dequant + matmul of the canonical FP4 weight as
the gold reference.

With ``--autotune``, the cuDNN (execution plans) and cute-dsl (tile
shape + perf-knob configs) backends are profiled; autotuned results are
tagged with an ``_autotune`` backend suffix.
"""
if args.verbose >= 1:
print("[INFO] Running testMmFp4W4A16")
print(f"[INFO] FlashInfer version: {flashinfer.__version__}")

device = get_device(args)
if args.generate_repro_command:
print(
f"[INFO] To reproduce this test case, run the following command: {args.repro_command}"
)

m, n, k = args.m, args.n, args.k
input_dtype = dtype_str_to_torch_dtype(args.input_dtype)
out_dtype = dtype_str_to_torch_dtype(args.out_dtype)
backends = args.backends
run_refcheck = args.refcheck
is_cuda_graph_compatible = not args.no_cuda_graph

if input_dtype != torch.bfloat16:
raise ValueError(
f"mm_fp4_w4a16 benchmark requires input_dtype=bfloat16, got {args.input_dtype}"
)
if out_dtype not in (torch.bfloat16, torch.float16):
raise ValueError(
f"mm_fp4_w4a16 benchmark requires out_dtype in (bfloat16, float16), got {args.out_dtype}"
)
if n % 64 != 0:
# N must be a multiple of the 64-wide N tile used by the weight
# prepack and the cute-dsl kernel. The 128x4 SF swizzle only *pads*
# N to 128, and prepare_w4a16_fp4_weights unswizzles the padded
# tail, so any n % 64 == 0 works.
raise ValueError("mm_fp4_w4a16 benchmark requires n % 64 == 0")
if k % 16 != 0:
raise ValueError("mm_fp4_w4a16 benchmark requires k % 16 == 0 (FP4 block size)")

torch.manual_seed(args.random_seed)
a = torch.randn((m, k), device=device, dtype=input_dtype) * 0.5
w = torch.randn((n, k), device=device, dtype=input_dtype) * 0.1
g_b = (448 * 6) / w.float().abs().nan_to_num().max()
b_fp4, b_sf = flashinfer.nvfp4_quantize(
w,
g_b,
sfLayout=flashinfer.SfLayout.layout_128x4,
do_shuffle=False,
backend="cute-dsl",
)
alpha = torch.tensor([1.0 / g_b.item()], device=device, dtype=torch.float32)

if args.verbose >= 2:
print(f"[VVERBOSE] {a.shape = } {a.dtype = }")
print(f"[VVERBOSE] {b_fp4.shape = } {b_fp4.dtype = }")
print(f"[VVERBOSE] {b_sf.shape = } {b_sf.dtype = }")

# Per-backend prep + runner closures. Prep is one-shot and not timed.
backend_runners = {}

def make_runner(b_p, sf_p, alpha_p, backend):
def run(a):
# W4A16 weight-only mode of mm_fp4: bf16 activation a, no
# a_descale; b_p/sf_p are prepare_w4a16_fp4_weights outputs.
return flashinfer.mm_fp4(
a,
b_p,
None,
sf_p,
alpha_p,
backend=backend,
out_dtype=out_dtype,
block_size=16,
enable_pdl=args.enable_pdl,
)

return run

backends_to_remove = []
for backend in backends:
try:
b_p, sf_p, alpha_p = flashinfer.prepare_w4a16_fp4_weights(
b_fp4, b_sf, alpha, backend=backend
)
runner = make_runner(b_p, sf_p, alpha_p, backend)
runner(a)
backend_runners[backend] = runner
except Exception as e:
print(
f"[INFO] {backend} backend does not support this configuration: {type(e).__name__}: {e}"
)
backends_to_remove.append(backend)

for backend in backends_to_remove:
backends.remove(backend)

if len(backends) == 0:
print("[ERROR] No backends passed validation. Exiting.")
return
Comment thread
bkryu marked this conversation as resolved.
Resolved

# cuDNN sweeps its execution plans + M buckets; cute-dsl sweeps its tile
# shape + perf-knob config space (both via mm_fp4 W4A16 TunableRunners).
autotune_supported_backends = ["cudnn", "cute-dsl"]
cache_path = getattr(args, "autotune_cache", None)
if getattr(args, "autotune", False):
warmup_iters = (
args.dry_run_iters if args.dry_run_iters and args.dry_run_iters > 0 else 10
)
for cur_backend in backends:
if cur_backend in autotune_supported_backends:
if args.verbose >= 1:
print(
f"[INFO] Autotune warmup for mm_fp4_w4a16 {cur_backend}: "
f"{warmup_iters} iters"
)
with autotune(True, cache=cache_path):
for _ in range(warmup_iters):
backend_runners[cur_backend](a)
elif cache_path:
with autotune(False, cache=cache_path):
pass

# Gold reference: fp32 dequant of the canonical FP4 weight + matmul, cast
# to out_dtype (the same math the removed "torch" backend computed).
ref = None
if run_refcheck:
weight_fp32 = _dequantize_w4a16_fp4_ref(b_fp4, b_sf, alpha, n, k, 16)
ref = (a.float() @ weight_fp32.T).to(out_dtype)

res = []
flops = 2 * m * n * k
bytes_accessed = (
m * k * input_dtype.itemsize
+ (k // 2) * n # FP4 weight (uint8, 2 codes / byte)
+ (k // 16) * n # FP8-E4M3 per-block SF
+ m * n * out_dtype.itemsize
)
Comment on lines +1500 to +1505

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

medium

Creating a dummy empty tensor torch.tensor([], dtype=input_dtype) just to call .element_size() is verbose and slightly inefficient. We can use a.element_size() and torch.empty((), dtype=out_dtype).element_size() instead.

Suggested change
bytes_accessed = (
m * k * torch.tensor([], dtype=input_dtype).element_size()
+ (k // 2) * n # FP4 weight (uint8, 2 codes / byte)
+ (k // 16) * n # FP8-E4M3 per-block SF
+ m * n * torch.tensor([], dtype=out_dtype).element_size()
)
bytes_accessed = (
m * k * a.element_size()
+ (k // 2) * n # FP4 weight (uint8, 2 codes / byte)
+ (k // 16) * n # FP8-E4M3 per-block SF
+ m * n * torch.empty((), dtype=out_dtype).element_size()
)

Copy link
Copy Markdown
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Done β€” went with input_dtype.itemsize / out_dtype.itemsize rather than the suggested torch.empty(()), to match the idiom used elsewhere in this file (e.g. the mm_fp4/mm_bf16 routines). No allocation at all and the computed value is unchanged.

# Refcheck tolerance vs the fp32-accurate gold. The cute-dsl and cudnn
# kernels dequantize the FP4 weight to bf16 before the tensor-core matmul,
# so they carry ~1 bf16 ULP of weight rounding (up to ~1.5e-2 relative)
# that the fp32 reference does not.
refcheck_tol = dict(rtol=1.5e-2, atol=1.5e-2)
for backend in backends:
runner = backend_runners[backend]
if run_refcheck:
out = runner(a)
try:
torch.testing.assert_close(out, ref, **refcheck_tol)
except AssertionError as e:
if args.allow_output_mismatch:
print(f"[WARNING] {backend} output mismatch vs fp32 ref: {e}")
else:
raise

timing = bench_gpu_time(
fn=runner,
dry_run_iters=args.dry_run_iters,
repeat_iters=args.num_iters,
sleep_after_run=True, # GEMMs are very MMA-heavy, so prefer sleep to reduce throttling.
enable_cupti=args.use_cupti,
use_cuda_graph=is_cuda_graph_compatible,
cold_l2_cache=True,
input_args=(a,),
)
Comment thread
coderabbitai[bot] marked this conversation as resolved.
Resolved
median_time = float(np.median(timing))
std_time = float(np.std(timing))
tflops = flops / median_time / 1e9
tb_per_sec = bytes_accessed / median_time / 1e9
backend_name = backend + (
"_autotune"
if (
getattr(args, "autotune", False)
and backend in autotune_supported_backends
)
else ""
)
print_perf_metrics(backend_name, median_time, std_time, tflops, tb_per_sec)
res.append(
{
"routine": args.routine,
"median_time": median_time,
"std_time": std_time,
"tflops": tflops,
"tb_per_sec": tb_per_sec,
"backend": backend_name,
"resolved_backend": backend,
"m": m,
"n": n,
"k": k,
"input_dtype": str(input_dtype).split(".")[-1],
"out_dtype": str(out_dtype).split(".")[-1],
"case_tag": args.case_tag,
}
)
return res


def testMmMxfp8(args):
"""
Test mm_mxfp8 API.
Expand Down
1 change: 1 addition & 0 deletions flashinfer/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -105,6 +105,7 @@
from .gemm import bmm_mxfp8 as bmm_mxfp8
from .gemm import mm_bf16 as mm_bf16
from .gemm import mm_fp4 as mm_fp4
from .gemm import prepare_w4a16_fp4_weights as prepare_w4a16_fp4_weights
from .gemm import mm_fp8 as mm_fp8
from .gemm import mm_mxfp8 as mm_mxfp8
from .gemm import tgv_gemm_sm100 as tgv_gemm_sm100
Expand Down
56 changes: 56 additions & 0 deletions flashinfer/cute_dsl/fp4_common.py
Original file line number Diff line number Diff line change
Expand Up @@ -283,6 +283,33 @@ def get_smem_ptr_as_int32(
return elem_ptr.toint(loc=loc, ip=ip)


@dsl_user_op
def ld_shared_v2_u32(smem_addr: Int32, *, loc=None, ip=None) -> Tuple[Uint32, Uint32]:
"""Load 64 bits (2 x uint32) from shared memory via ld.shared.v2.u32.

Args:
smem_addr: 32-bit shared memory address (from get_smem_ptr_as_int32).
Caller is responsible for ensuring 8-byte alignment.

Returns:
2 Uint32 values (8 bytes total).
"""
result = llvm.inline_asm(
llvm.StructType.get_literal([T.i32(), T.i32()]),
[Int32(smem_addr).ir_value(loc=loc, ip=ip)],
"ld.shared.v2.u32 {$0, $1}, [$2];",
"=r,=r,r",
has_side_effects=False,
is_align_stack=False,
asm_dialect=llvm.AsmDialect.AD_ATT,
loc=loc,
ip=ip,
)
v0 = llvm.extractvalue(T.i32(), result, [0], loc=loc, ip=ip)
v1 = llvm.extractvalue(T.i32(), result, [1], loc=loc, ip=ip)
return Uint32(v0), Uint32(v1)


@dsl_user_op
def ld_shared_v4_u32(
smem_addr: Int32, *, loc=None, ip=None
Expand Down Expand Up @@ -1996,6 +2023,35 @@ def cvt_e4m3_to_f32_via_f16(fp8_val: Uint32, *, loc=None, ip=None) -> Float32:
)


@dsl_user_op
def cvt_s0e5m3_to_f16x2_broadcast(fp8_val: Uint32, *, loc=None, ip=None) -> Uint32:
"""Convert one S0E5M3 scale byte to an f16x2 with the scale in both lanes.

S0E5M3 (sign-0, 5-exp, 3-mantissa) is a host-side reformat of the per-block
E4M3 scale, rebiased to fp16's bias (exp 7->15, i.e. byte += 0x40) so the
bits line up with fp16 directly: ``f16(byte) = byte << 7`` (exp -> bits 14-10,
the 3 mantissa bits -> bits 9-7, low mantissa and sign = 0). Broadcasting to
both f16x2 lanes is then ``byte * 0x00800080`` = ``(byte<<7) | (byte<<23)`` --
a single ``mul.lo.u32`` (the two shifted copies never overlap, so the mul is
exactly the OR). Replaces the 3-op E4M3 path (cvt.u16 + cvt.f16x2.e4m3x2 +
prmt) with 1 op, and the per-block scale stays 1 byte (memory-neutral).
Numerically exact for normal E4M3 scales (both formats carry 3 mantissa bits).
"""
return Uint32(
llvm.inline_asm(
T.i32(),
[Uint32(fp8_val).ir_value(loc=loc, ip=ip)],
"mul.lo.u32 $0, $1, 0x00800080;",
"=r,r",
has_side_effects=False,
is_align_stack=False,
asm_dialect=llvm.AsmDialect.AD_ATT,
loc=loc,
ip=ip,
)
)


@dsl_user_op
def fp4_decode_4bytes(
packed_u32: Uint32, *, loc=None, ip=None
Expand Down
5 changes: 5 additions & 0 deletions flashinfer/gemm/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,10 @@
from .gemm_base import group_gemm_fp8_nt_groupwise as group_gemm_fp8_nt_groupwise
from .gemm_base import fp8_blockscale_gemm_sm90 as fp8_blockscale_gemm_sm90

from .gemm_w4a16 import (
prepare_w4a16_fp4_weights as prepare_w4a16_fp4_weights,
)

from .routergemm import (
mm_M1_16_K6144_N256 as mm_M1_16_K6144_N256,
mm_M1_16_K7168_N128 as mm_M1_16_K7168_N128,
Expand Down Expand Up @@ -101,6 +105,7 @@
"gemm_fp8_nt_groupwise",
"group_gemm_fp8_nt_groupwise",
"fp8_blockscale_gemm_sm90",
"prepare_w4a16_fp4_weights",

Copy link
Copy Markdown
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Is this flashinfer api?

Copy link
Copy Markdown
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yes this is a new API being introduced in the PR and it is meant to be a performance optimization. The PR description explains this

API
Weight-only quantized GEMMs need a backend-specific weight layout, so the API is a two-step flow:

  • prepare_w4a16_fp4_weights --> For preprocessing nvfp4 weights (B operand) The preparation step is a performance optimization to accelerate the dequantization at GEMM runtime.
  • mm_fp4(..., a_descale=None) β†’ execute the W4A16 GEMM. backend="auto" resolves the backend from the prepared weight layout (uint8 β†’ cudnn, int32 β†’ cute-dsl).

Along with an example usage. Because the dequantization adds a significant overhead to a BF16 GEMM, pre-processing (e.g. unswizzling the swizzled weights or converting the S1E4M3 scale factor to S0E5M3) ahead of time reduces the number of FMA/ALU operations during GEMM runtime.

Having a preparation stage is reasonable in the W4A16 workloads is acceptable because the W4 will be the model weights during e2e inference. When the model is loaded, the preparation function can be called for the dense GEMM weights to do this offloading of dequantization. In fact, vLLM's Marlin backend for W4A16 has it's own special layout that requires a very similar preparation function call during model load time, but the exact layout is different

"mm_M1_16_K6144_N256",
"mm_M1_16_K7168_N128",
"mm_M1_16_K7168_N256",
Expand Down
Loading
Loading