diff --git a/benchmarks/bench_vsa_sage_sm120.py b/benchmarks/bench_vsa_sage_sm120.py new file mode 100644 index 00000000000..bfbed82ee3f --- /dev/null +++ b/benchmarks/bench_vsa_sage_sm120.py @@ -0,0 +1,142 @@ +""" +Benchmark: SM120 Sage (QK-INT8/PV-FP8) block-sparse VSA attention. + +Benchmark configurations: + b=8, h=32, head_dim=128 + seqlen: 1024, 2048, 4096, 8192, 16384, 32768, 65536 + density: 10%, 50%, 90% + +Usage: + python benchmarks/bench_vsa_sage_sm120.py +""" + +import statistics +import sys + +import torch + +from flashinfer.cute_dsl.sparse.bsa_attn_sm120 import bsa_attn_sm120_blk64_sage_fwd +from flashinfer.cute_dsl.sparse.bsa_utils.sage_quant_sm120 import ( + quantize_sage_qkv_sm120, +) +from flashinfer.testing import bench_gpu_time + + +BLOCK = 64 +HEAD_DIM = 128 + + +def _build_q2k(batch, heads, num_q_blocks, num_kv_blocks, density, device): + capacity = num_kv_blocks + index = torch.zeros( + batch, heads, num_q_blocks, capacity, dtype=torch.int32, device=device + ) + nums = torch.zeros(batch, heads, num_q_blocks, dtype=torch.int32, device=device) + for b in range(batch): + for h in range(heads): + for qi in range(num_q_blocks): + k = max(1, int(round(density * num_kv_blocks))) + k = min(k, num_kv_blocks) + chosen = torch.randperm(num_kv_blocks, device=device)[:k].sort().values + index[b, h, qi, :k] = chosen.to(torch.int32) + nums[b, h, qi] = k + return index, nums + + +def run_benchmark(): + device = torch.device("cuda") + if torch.cuda.get_device_capability(device) != (12, 0): + print("ERROR: SM120 GPU (compute capability 12.0) required.") + sys.exit(1) + + torch.manual_seed(42) + + batch = 8 + num_heads = 32 + seqlens = [1024, 2048, 4096, 8192, 16384, 32768, 65536] + densities = [0.10, 0.50, 0.90] + + col_w = [8, 9, 12, 11, 10] + header = ( + f"{'seqlen':>{col_w[0]}} {'density':>{col_w[1]}} " + f"{'active_blks':>{col_w[2]}} {'median_ms':>{col_w[3]}} {'tflops':>{col_w[4]}}" + ) + sep = "-" * len(header) + print(f"\nSM120 Sage VSA b={batch} h={num_heads} head_dim={HEAD_DIM}") + print(header) + print(sep) + + for seqlen in seqlens: + num_blocks = seqlen // BLOCK + + q_bf16 = torch.randn( + batch, num_heads, seqlen, HEAD_DIM, dtype=torch.bfloat16, device=device + ) + k_bf16 = torch.randn( + batch, num_heads, seqlen, HEAD_DIM, dtype=torch.bfloat16, device=device + ) + v_bf16 = torch.randn( + batch, num_heads, seqlen, HEAD_DIM, dtype=torch.bfloat16, device=device + ) + + q_int8, k_int8, v_fp8, q_scale, k_scale, v_scale = quantize_sage_qkv_sm120( + q_bf16, k_bf16, v_bf16 + ) + + for density in densities: + q2k_index, q2k_nums = _build_q2k( + batch, num_heads, num_blocks, num_blocks, density, device + ) + active_blocks = int(q2k_nums.sum().item()) + + # warm-up + bsa_attn_sm120_blk64_sage_fwd( + q_int8, + k_int8, + v_fp8, + q_scale, + k_scale, + v_scale, + q2k_index, + block_sparse_num=int(q2k_nums.max().item()), + q2k_block_nums=q2k_nums, + backend="cute_dsl", + ) + torch.cuda.synchronize() + + times = bench_gpu_time( + lambda: bsa_attn_sm120_blk64_sage_fwd( + q_int8, + k_int8, + v_fp8, + q_scale, + k_scale, + v_scale, + q2k_index, + block_sparse_num=int(q2k_nums.max().item()), + q2k_block_nums=q2k_nums, + backend="cute_dsl", + ), + repeat_time_ms=500, + ) + ms = statistics.median(times) + + # FLOPs: active_blocks includes batch*heads; each (q_tile, k_tile) pair + # does two GEMMs: QK (2*BLOCK*HEAD_DIM*BLOCK) + PV (2*BLOCK*BLOCK*HEAD_DIM) + # = 4*BLOCK^2*HEAD_DIM per active block. + flops = 4 * active_blocks * BLOCK * BLOCK * HEAD_DIM + tflops = flops / (ms * 1e-3) / 1e12 + actual_density = active_blocks / ( + num_blocks * num_blocks * batch * num_heads + ) + + print( + f"{seqlen:>{col_w[0]}} {actual_density:>{col_w[1]}.3f} " + f"{active_blocks:>{col_w[2]}} {ms:>{col_w[3]}.3f} {tflops:>{col_w[4]}.2f}" + ) + + print(sep) + + +if __name__ == "__main__": + run_benchmark() diff --git a/flashinfer/cute_dsl/sparse/bsa_attn_sm120.py b/flashinfer/cute_dsl/sparse/bsa_attn_sm120.py index c9fcd1802a7..0dd075049c8 100644 --- a/flashinfer/cute_dsl/sparse/bsa_attn_sm120.py +++ b/flashinfer/cute_dsl/sparse/bsa_attn_sm120.py @@ -52,37 +52,75 @@ def _prepare_sm120_sparse_metadata( device: torch.device, ): """Validate and transpose SM120 sparse metadata into kernel ABI layout.""" - assert q2k_block_index.dtype == torch.int32 - assert q2k_block_index.device == device - assert q2k_block_index.shape[:3] == (batch_size, num_heads, num_q_blocks) + if q2k_block_index.dtype != torch.int32: + raise TypeError(f"q2k_block_index must be int32, got {q2k_block_index.dtype}") + if q2k_block_index.device != device: + raise ValueError( + f"q2k_block_index must be on {device}, got {q2k_block_index.device}" + ) + if q2k_block_index.shape[:3] != (batch_size, num_heads, num_q_blocks): + raise ValueError( + f"q2k_block_index.shape {tuple(q2k_block_index.shape)} must start with " + f"(batch={batch_size}, num_heads={num_heads}, num_q_blocks={num_q_blocks})" + ) has_block_nums = q2k_block_nums is not None and q2k_block_nums.numel() > 0 if has_block_nums: - assert q2k_block_nums.dtype == torch.int32 - assert q2k_block_nums.device == device - assert q2k_block_nums.shape == (batch_size, num_heads, num_q_blocks) + if q2k_block_nums.dtype != torch.int32: + raise TypeError(f"q2k_block_nums must be int32, got {q2k_block_nums.dtype}") + if q2k_block_nums.device != device: + raise ValueError( + f"q2k_block_nums must be on {device}, got {q2k_block_nums.device}" + ) + if q2k_block_nums.shape != (batch_size, num_heads, num_q_blocks): + raise ValueError( + f"q2k_block_nums.shape {tuple(q2k_block_nums.shape)} must be " + f"(batch={batch_size}, num_heads={num_heads}, num_q_blocks={num_q_blocks})" + ) q2k_block_nums = q2k_block_nums.contiguous() else: - assert 1 <= block_sparse_num <= q2k_block_index.shape[-1] + capacity = q2k_block_index.shape[-1] + if not (0 <= block_sparse_num <= capacity): + raise ValueError( + f"block_sparse_num ({block_sparse_num}) must be in [0, capacity={capacity}]" + ) has_block_sizes = block_sizes is not None and block_sizes.numel() > 0 block_sizes_mode = 0 if has_block_sizes: - assert block_sizes.dtype == torch.int32 - assert block_sizes.device == device + if block_sizes.dtype != torch.int32: + raise TypeError(f"block_sizes must be int32, got {block_sizes.dtype}") + if block_sizes.device != device: + raise ValueError( + f"block_sizes must be on {device}, got {block_sizes.device}" + ) if block_sizes.ndim == 1: - assert block_sizes.shape == (num_kv_blocks,) + if block_sizes.shape != (num_kv_blocks,): + raise ValueError( + f"block_sizes.shape {tuple(block_sizes.shape)} must be ({num_kv_blocks},)" + ) block_sizes_t = block_sizes.contiguous() block_sizes_mode = 1 elif block_sizes.ndim == 2: - assert block_sizes.shape == (batch_size, num_kv_blocks) + if block_sizes.shape != (batch_size, num_kv_blocks): + raise ValueError( + f"block_sizes.shape {tuple(block_sizes.shape)} must be " + f"({batch_size}, {num_kv_blocks})" + ) block_sizes_t = block_sizes.contiguous().permute(1, 0) block_sizes_mode = 2 - else: - assert block_sizes.ndim == 3 - assert block_sizes.shape == (batch_size, num_heads, num_kv_blocks) + elif block_sizes.ndim == 3: + if block_sizes.shape != (batch_size, num_heads, num_kv_blocks): + raise ValueError( + f"block_sizes.shape {tuple(block_sizes.shape)} must be " + f"({batch_size}, {num_heads}, {num_kv_blocks})" + ) block_sizes_t = block_sizes.contiguous().permute(2, 1, 0) block_sizes_mode = 3 + else: + raise ValueError( + f"block_sizes must be 1D, 2D, or 3D, got {block_sizes.ndim}D" + ) # (B, H, Q, K) -> (K, Q, H, B) q2k_t = q2k_block_index.contiguous().permute(3, 2, 1, 0) @@ -511,6 +549,9 @@ def _bsa_attn_sm120_blk64_sage_fwd_cake( return out +_sm120_sage_compile_cache = get_jit_cache("bsa_fwd_sm120_sage") + + @flashinfer_api def bsa_attn_sm120_blk64_sage_fwd( q_int8: torch.Tensor, @@ -525,19 +566,33 @@ def bsa_attn_sm120_blk64_sage_fwd( q2k_block_nums: Optional[torch.Tensor] = None, softmax_scale: Optional[float] = None, *, - out: torch.Tensor, - tma_descriptor_workspace: torch.Tensor, + out: Optional[torch.Tensor] = None, + tma_descriptor_workspace: Optional[torch.Tensor] = None, uniform_block_count: bool = False, contiguous_block_indices: bool = False, backend: str = "cake", ) -> torch.Tensor: """Run prequantized SM120 Sage block-sparse attention. - All CUDA storage is caller-owned. ``out`` is contiguous BF16 BHSD. - Tensor-map descriptors are passed by value as grid-constant kernel - parameters; ``tma_descriptor_workspace`` is accepted but unused. - Q/K use contiguous INT8 BHSD, V uses contiguous FP8 E4M3 HDS, and the - operation supports MHA, head dimension 128, non-causal forward without LSE. + Supports two backends selected via ``backend``: + + ``"cake"`` (default) + tvm-ffi generated kernel. All CUDA storage is caller-owned: ``out`` + and ``tma_descriptor_workspace`` are required keyword arguments. + Tensor-map descriptors are passed by value as grid-constant kernel + parameters; ``tma_descriptor_workspace`` is accepted but unused. + ``uniform_block_count`` and ``contiguous_block_indices`` tuning flags + are only honoured by this backend. + + ``"cute_dsl"`` + CuTe-DSL port of upstream Block-Sparse-Attention. Uses BHSD tensor + layout throughout, allocates ``out`` internally when not provided, + and does not require ``tma_descriptor_workspace``. Supports SM120 + only (exact compute capability 12.0), MHA only, head dimension 128, + non-causal forward without LSE. ``uniform_block_count`` and + ``contiguous_block_indices`` are silently ignored by this backend. + + Q/K use contiguous INT8 BHSD, V uses contiguous FP8 E4M3 HDS. Parameters ---------- @@ -570,30 +625,39 @@ def bsa_attn_sm120_blk64_sage_fwd( ``[B, H, ceil(Sq / 64)]``. softmax_scale : float, optional Positive finite softmax scale. ``None`` selects ``1 / sqrt(128)``. - out : torch.Tensor - Caller-owned contiguous BF16 output with shape ``[B, H, Sq, 128]``. - tma_descriptor_workspace : torch.Tensor + out : torch.Tensor, optional + Pre-allocated contiguous BF16 output with shape ``[B, H, Sq, 128]``. + Required for ``backend="cake"``. For ``backend="cute_dsl"``, + allocated internally when not provided. + tma_descriptor_workspace : torch.Tensor, optional Accepted for compatibility and unused. An empty tensor is sufficient; this operation neither reads nor writes its storage. Tensor-map descriptors are passed by value in the kernel launch, so workspace contents, lifetime, and reuse across streams do not affect attention. + Required for ``backend="cake"``; ignored for ``backend="cute_dsl"``. uniform_block_count : bool Whether every query block uses ``block_sparse_num`` selected blocks. - This is a caller-provided guarantee: when true, ``q2k_block_nums`` - is ignored and its values are not checked. + Only honoured by ``backend="cake"``, where it is a caller-provided + guarantee: when true, ``q2k_block_nums`` is ignored and its values + are not checked. Silently ignored by ``backend="cute_dsl"``. contiguous_block_indices : bool - Whether selected block indices are contiguous. This optimization - requires ``uniform_block_count=True`` and trusts the caller's - guarantee without checking the indices. Non-contiguous top-k indices - must use ``False``, even when sorted. Incorrectly setting this flag - to ``True`` can produce incorrect output without raising an error. + Whether selected block indices are contiguous. Requires + ``uniform_block_count=True``. Only honoured by ``backend="cake"``, + which trusts the caller's guarantee without checking the indices; + non-contiguous top-k indices must use ``False``, even when sorted. + Incorrectly setting this flag to ``True`` under ``backend="cake"`` + can produce incorrect output without raising an error. Silently + ignored by ``backend="cute_dsl"``. backend : str - Backend name. The only supported value is ``"cake"``. + Backend name. ``"cake"`` (default) or ``"cute_dsl"``. Returns ------- torch.Tensor - The caller-owned ``out`` tensor after attention output is written. + The output tensor after attention output is written. For + ``backend="cake"`` this is always the caller-owned ``out``. For + ``backend="cute_dsl"``, ``out`` is returned directly when provided; + otherwise a newly allocated tensor is returned. Notes ----- @@ -601,32 +665,266 @@ def bsa_attn_sm120_blk64_sage_fwd( scalar ``block_sparse_num`` count, with every supported ``block_sizes`` layout and with aligned or partial final KV blocks. Omitting ``q2k_block_nums`` uses the scalar count even when - ``uniform_block_count=False``. Setting ``uniform_block_count=True`` - uses the scalar count and ignores any supplied per-row counts. A row - selecting zero blocks produces an exactly zero output. + ``uniform_block_count=False``. Under ``backend="cake"``, setting + ``uniform_block_count=True`` uses the scalar count and ignores any + supplied per-row counts; ``backend="cute_dsl"`` accepts the flag but + silently ignores it. A row selecting zero blocks produces an exactly + zero output. Initialize the JIT module before CUDA Graph capture. Keep captured input and output tensors alive while their graph is in use. Each captured launch retains its own tensor-map descriptors, so launches with different tensor bindings can be interleaved without shared descriptor storage. """ - if backend != "cake": + if backend == "cake": + if out is None: + raise ValueError("backend='cake' requires a caller-owned out tensor") + if tma_descriptor_workspace is None: + raise ValueError("backend='cake' requires tma_descriptor_workspace") + return _bsa_attn_sm120_blk64_sage_fwd_cake( + q_int8, + k_int8, + v_fp8, + q_scale, + k_scale, + v_scale, + q2k_block_index, + block_sparse_num, + block_sizes, + q2k_block_nums, + softmax_scale, + out=out, + tma_descriptor_workspace=tma_descriptor_workspace, + uniform_block_count=uniform_block_count, + contiguous_block_indices=contiguous_block_indices, + ) + elif backend != "cute_dsl": raise ValueError(f"unsupported SM120 Sage backend: {backend!r}") - return _bsa_attn_sm120_blk64_sage_fwd_cake( - q_int8, - k_int8, - v_fp8, - q_scale, - k_scale, - v_scale, + + from .sm120_blk64.flash_fwd_sm120_sage import ( # noqa: PLC0415 + BlockSparseAttnForwardSageSm120Blk64, + ) + + def _require_contiguous_cuda(name: str, tensor: torch.Tensor, device) -> None: + if not tensor.is_cuda: + raise ValueError(f"{name} must be a CUDA tensor") + if tensor.device != device: + raise ValueError(f"{name} must be on {device}, got {tensor.device}") + if not tensor.is_contiguous(): + raise ValueError(f"{name} must be contiguous") + + if q_int8.dtype != torch.int8 or k_int8.dtype != torch.int8: + raise TypeError( + f"bsa_attn_sm120_blk64_sage_fwd requires int8 Q/K, " + f"got q_int8={q_int8.dtype}, k_int8={k_int8.dtype}" + ) + if v_fp8.dtype != torch.float8_e4m3fn: + raise TypeError( + f"bsa_attn_sm120_blk64_sage_fwd requires float8_e4m3fn V, got {v_fp8.dtype}" + ) + if q_int8.dim() != 4 or k_int8.dim() != 4 or v_fp8.dim() != 4: + raise ValueError("q_int8, k_int8, and v_fp8 must be rank-4 BHSD tensors") + + _require_contiguous_cuda("q_int8", q_int8, q_int8.device) + _require_contiguous_cuda("k_int8", k_int8, q_int8.device) + _require_contiguous_cuda("v_fp8", v_fp8, q_int8.device) + _require_contiguous_cuda("q_scale", q_scale, q_int8.device) + _require_contiguous_cuda("k_scale", k_scale, q_int8.device) + _require_contiguous_cuda("v_scale", v_scale, q_int8.device) + for name, tensor in ( + ("q_scale", q_scale), + ("k_scale", k_scale), + ("v_scale", v_scale), + ): + if tensor.dtype != torch.float32: + raise ValueError(f"{name} must use torch.float32, got {tensor.dtype}") + + capability = tuple(int(v) for v in torch.cuda.get_device_capability(q_int8.device)) + if capability != (12, 0): + raise RuntimeError( + f"bsa_attn_sm120_blk64_sage_fwd (cute_dsl) requires SM120 (12.0), " + f"got {capability}" + ) + arch = capability[0] * 10 + capability[1] + + batch, num_heads, seqlen_q, head_dim = q_int8.shape + seqlen_k = k_int8.shape[2] + + if head_dim != 128: + raise ValueError(f"sm120_blk64 Sage requires head_dim=128, got {head_dim}") + if k_int8.shape != (batch, num_heads, seqlen_k, head_dim): + raise ValueError( + f"k_int8.shape {tuple(k_int8.shape)} must be " + f"(batch={batch}, num_heads={num_heads}, seqlen_k={seqlen_k}, head_dim={head_dim}) " + "(only MHA is supported: num_kv_heads must equal num_heads)" + ) + + padded_k = _ceil_div(seqlen_k, _BLOCK_SIZE) * _BLOCK_SIZE + if v_fp8.shape != (batch, num_heads, head_dim, padded_k): + raise ValueError( + f"v_fp8.shape {tuple(v_fp8.shape)} must be " + f"(batch={batch}, num_heads={num_heads}, head_dim={head_dim}, " + f"padded_seqlen_k={padded_k})" + ) + + num_q_blocks = _ceil_div(seqlen_q, _BLOCK_SIZE) + num_kv_blocks = _ceil_div(seqlen_k, _BLOCK_SIZE) + + num_q_scale_groups = _ceil_div(seqlen_q, 128) * 4 + for name, tensor, expected_shape in ( + ("q_scale", q_scale, (batch, num_heads, num_q_scale_groups)), + ("k_scale", k_scale, (batch, num_heads, num_kv_blocks)), + ("v_scale", v_scale, (batch, num_heads, head_dim)), + ): + if tuple(tensor.shape) != expected_shape: + raise ValueError( + f"{name}.shape {tuple(tensor.shape)} must be {expected_shape}" + ) + + if softmax_scale is None: + softmax_scale = head_dim**-0.5 + if isinstance(softmax_scale, bool) or not isinstance(softmax_scale, (int, float)): + raise TypeError("softmax_scale must be a number") + if not math.isfinite(float(softmax_scale)) or float(softmax_scale) <= 0.0: + raise ValueError("softmax_scale must be finite and positive") + + if q_int8.device.index != torch.cuda.current_device(): + raise ValueError( + f"q_int8 is on cuda:{q_int8.device.index} but the current CUDA device is " + f"cuda:{torch.cuda.current_device()}; inputs must reside on the current device" + ) + + if out is not None: + if out.dtype != torch.bfloat16: + raise TypeError(f"out.dtype ({out.dtype}) must be torch.bfloat16") + if out.shape != (batch, num_heads, seqlen_q, head_dim): + raise ValueError( + f"out.shape {tuple(out.shape)} must be " + f"(batch={batch}, num_heads={num_heads}, seqlen_q={seqlen_q}, head_dim={head_dim})" + ) + if out.device != q_int8.device: + raise ValueError(f"out must be on {q_int8.device}, got {out.device}") + if not out.is_contiguous(): + raise ValueError("out must be contiguous") + else: + out = torch.empty( + (batch, num_heads, seqlen_q, head_dim), + dtype=torch.bfloat16, + device=q_int8.device, + ) + + ( + has_block_nums, + has_block_sizes, + block_sizes_mode, + q2k_t, + q2k_nums_t, + block_sizes_t, + ) = _prepare_sm120_sparse_metadata( q2k_block_index, - block_sparse_num, - block_sizes, q2k_block_nums, - softmax_scale, - out=out, - tma_descriptor_workspace=tma_descriptor_workspace, - uniform_block_count=uniform_block_count, - contiguous_block_indices=contiguous_block_indices, - backend="cake", + block_sizes, + block_sparse_num, + batch_size=batch, + num_heads=num_heads, + num_q_blocks=num_q_blocks, + num_kv_blocks=num_kv_blocks, + device=q_int8.device, + ) + + # BHSD -> kernel layout: Q/K/O (B,H,S,D) -> (S,D,H,B), leading_dim=1. + # V is already Sage's (B,H,D,padded_S) -> (D,S,H,B), leading_dim=1. + # Scales (B,H,G) -> (G,H,B), leading_dim=0. + q_t = q_int8.permute(2, 3, 1, 0) + k_t = k_int8.permute(2, 3, 1, 0) + v_t = v_fp8.permute(2, 3, 1, 0) + out_t = out.permute(2, 3, 1, 0) + q_scale_t = q_scale.permute(2, 1, 0) + k_scale_t = k_scale.permute(2, 1, 0) + v_scale_t = v_scale.permute(2, 1, 0) + + q_cute = to_cute_tensor(q_t, assumed_align=128, leading_dim=1, enable_tvm_ffi=False) + k_cute = to_cute_tensor(k_t, assumed_align=128, leading_dim=1, enable_tvm_ffi=False) + v_cute = to_cute_tensor(v_t, assumed_align=128, leading_dim=1, enable_tvm_ffi=False) + out_cute = to_cute_tensor( + out_t, assumed_align=128, leading_dim=1, enable_tvm_ffi=False + ) + q_scale_cute = to_cute_tensor( + q_scale_t, assumed_align=4, leading_dim=0, enable_tvm_ffi=False + ) + k_scale_cute = to_cute_tensor( + k_scale_t, assumed_align=4, leading_dim=0, enable_tvm_ffi=False + ) + v_scale_cute = to_cute_tensor( + v_scale_t, assumed_align=4, leading_dim=0, enable_tvm_ffi=False + ) + q2k_cute = to_cute_tensor( + q2k_t, assumed_align=None, leading_dim=0, enable_tvm_ffi=False ) + q2k_nums_cute = to_cute_tensor( + q2k_nums_t, assumed_align=None, leading_dim=0, enable_tvm_ffi=False + ) + block_sizes_cute = to_cute_tensor( + block_sizes_t, assumed_align=None, leading_dim=0, enable_tvm_ffi=False + ) + + current_stream = ( + cute.runtime.make_fake_stream(use_tvm_ffi_env_stream=True) + if is_fake_mode() + else cuda.CUstream(torch.cuda.current_stream().cuda_stream) + ) + + fwd_kernel = BlockSparseAttnForwardSageSm120Blk64( + gqa_ratio=1, + head_dim=head_dim, + value_dim=head_dim, + blocksparse_blocksize_q=_BLOCK_SIZE, + blocksparse_blocksize_k=_BLOCK_SIZE, + has_block_sizes=has_block_sizes, + has_block_nums=has_block_nums, + block_sizes_mode=block_sizes_mode, + ) + + compile_key = ( + "sm120_blk64_sage_fwd", + int(arch), + int(head_dim), + bool(has_block_nums), + bool(has_block_sizes), + int(block_sizes_mode), + q_t.stride(), + k_t.stride(), + v_t.stride(), + out_t.stride(), + q_scale_t.stride(), + k_scale_t.stride(), + v_scale_t.stride(), + q2k_t.stride(), + q2k_nums_t.stride(), + block_sizes_t.stride(), + ) + + args = ( + q_cute, + k_cute, + v_cute, + out_cute, + q_scale_cute, + k_scale_cute, + v_scale_cute, + q2k_cute, + q2k_nums_cute, + cutlass.Int32(block_sparse_num), + block_sizes_cute, + cutlass.Float32(softmax_scale), + current_stream, + ) + + if compile_key not in _sm120_sage_compile_cache: + _sm120_sage_compile_cache[compile_key] = cute.compile(fwd_kernel, *args) + + if not is_fake_mode(): + with torch.cuda.nvtx.range("bsa_attn_sm120_blk64_sage_fwd_kernel"): + _sm120_sage_compile_cache[compile_key](*args) + + return out diff --git a/flashinfer/cute_dsl/sparse/bsa_utils/sage_quant_sm120.py b/flashinfer/cute_dsl/sparse/bsa_utils/sage_quant_sm120.py new file mode 100644 index 00000000000..68a3e47220c --- /dev/null +++ b/flashinfer/cute_dsl/sparse/bsa_utils/sage_quant_sm120.py @@ -0,0 +1,616 @@ +# Copyright (c) 2025 by FlashInfer team. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# Ported from Block-Sparse-Attention's csrc/fwd/sm120_blk64/bsa_quant_sm120_sage.py +# and bsa_sage_quant.py: Triton quantization kernels that produce the exact +# QK-INT8 / PV-FP8 operand layout consumed by +# flash_fwd_sm120_sage.BlockSparseAttnForwardSageSm120Blk64. +# +# No AOT-compiled quant runtime fallback and no external-workspace reuse +# parameter -- both are optional performance paths, not required for correctness. + +from typing import Optional, Sequence + +import torch +import triton +import triton.language as tl +from triton.language.extra import libdevice + +SAGE_Q_GROUP_SIZE = 32 +SAGE_Q_BLOCK_SIZE = 128 +SAGE_K_BLOCK_SIZE = 64 +SAGE_KV_STATS_CHUNK = 256 +SAGE_HEAD_DIM = 128 +# FP16 PV accumulator overflow constraint: the worst-case PV tile sum must +# stay below FP16_MAX (65504). With FP8-E4M3 max 448 and a K-tile of 64 +# tokens: 448 * SAGE_V_SCALE_MAX * 64 = 64512 < 65504. If SAGE_P_QUANT_SCALE +# (flash_fwd_sm120_sage.py) or the K-tile size changes, re-verify this bound. +SAGE_V_SCALE_MAX = 2.25 + + +# ============================================================================= +# Triton kernels +# ============================================================================= + + +@triton.jit +def _quantize_sage_q_kernel( + q_ptr, + q8_ptr, + scale_ptr, + q_stride_b, + q_stride_h, + q_stride_s, + q8_stride_b, + q8_stride_h, + q8_stride_s, + scale_stride_b, + scale_stride_h, + seqlen_q, + batch_size, + HEAD_DIM: tl.constexpr, + GROUP_SIZE: tl.constexpr, +): + group_idx = tl.program_id(0) + head_idx = tl.program_id(1) + batch_idx = tl.program_id(2) + + row_idx = group_idx * GROUP_SIZE + tl.arange(0, GROUP_SIZE) + dim_idx = tl.arange(0, HEAD_DIM) + valid = row_idx < seqlen_q + q = tl.load( + q_ptr + + batch_idx * q_stride_b + + head_idx * q_stride_h + + row_idx[:, None] * q_stride_s + + dim_idx[None, :], + mask=valid[:, None], + other=0.0, + ).to(tl.float32) + + row_max = tl.max(tl.abs(q), axis=1) + amax = tl.maximum(tl.max(row_max, axis=0), 1.0e-7) + scale = amax / 127.0 + q_quant = tl.maximum( + tl.minimum(libdevice.rint(q / scale), 127.0), + -127.0, + ) + tl.store( + q8_ptr + + batch_idx * q8_stride_b + + head_idx * q8_stride_h + + row_idx[:, None] * q8_stride_s + + dim_idx[None, :], + q_quant.to(tl.int8), + mask=valid[:, None], + ) + tl.store( + scale_ptr + batch_idx * scale_stride_b + head_idx * scale_stride_h + group_idx, + scale, + ) + + +@triton.jit +def _sage_kv_stats_partial_kernel( + k_ptr, + v_ptr, + k_partial_ptr, + v_partial_ptr, + k_stride_b, + k_stride_h, + k_stride_s, + v_stride_b, + v_stride_h, + v_stride_s, + partial_stride_b, + partial_stride_h, + partial_stride_c, + seqlen_k, + heads, + batch_size, + HEAD_DIM: tl.constexpr, + CHUNK_SIZE: tl.constexpr, + DIM_TILE: tl.constexpr, +): + chunk_idx = tl.program_id(0) + dim_tile_idx = tl.program_id(1) + batch_head_idx = tl.program_id(2) + batch_idx = batch_head_idx // heads + head_idx = batch_head_idx - batch_idx * heads + + row_idx = chunk_idx * CHUNK_SIZE + tl.arange(0, CHUNK_SIZE) + dim_idx = dim_tile_idx * DIM_TILE + tl.arange(0, DIM_TILE) + valid = row_idx < seqlen_k + k = tl.load( + k_ptr + + batch_idx * k_stride_b + + head_idx * k_stride_h + + row_idx[:, None] * k_stride_s + + dim_idx[None, :], + mask=valid[:, None], + other=0.0, + ).to(tl.float32) + v = tl.load( + v_ptr + + batch_idx * v_stride_b + + head_idx * v_stride_h + + row_idx[:, None] * v_stride_s + + dim_idx[None, :], + mask=valid[:, None], + other=0.0, + ).to(tl.float32) + + partial_base = ( + batch_idx * partial_stride_b + + head_idx * partial_stride_h + + chunk_idx * partial_stride_c + + dim_idx + ) + tl.store(partial_base + k_partial_ptr, tl.sum(k, axis=0)) + tl.store(partial_base + v_partial_ptr, tl.max(tl.abs(v), axis=0)) + + +@triton.jit +def _sage_kv_stats_finalize_kernel( + k_partial_ptr, + v_partial_ptr, + k_mean_ptr, + v_scale_ptr, + partial_stride_b, + partial_stride_h, + partial_stride_c, + mean_stride_b, + mean_stride_h, + scale_stride_b, + scale_stride_h, + num_chunks, + seqlen_k, + heads, + batch_size, + DIM_TILE: tl.constexpr, + REDUCE_TILE: tl.constexpr, + SCALE_MAX: tl.constexpr, +): + dim_tile_idx = tl.program_id(0) + batch_head_idx = tl.program_id(1) + batch_idx = batch_head_idx // heads + head_idx = batch_head_idx - batch_idx * heads + dim_idx = dim_tile_idx * DIM_TILE + tl.arange(0, DIM_TILE) + + sum_acc = tl.zeros((DIM_TILE,), tl.float32) + max_acc = tl.zeros((DIM_TILE,), tl.float32) + chunk_base = 0 + while chunk_base < num_chunks: + chunk_idx = chunk_base + tl.arange(0, REDUCE_TILE) + mask = chunk_idx < num_chunks + partial_offset = ( + batch_idx * partial_stride_b + + head_idx * partial_stride_h + + chunk_idx[:, None] * partial_stride_c + + dim_idx[None, :] + ) + partial_sum = tl.load( + k_partial_ptr + partial_offset, + mask=mask[:, None], + other=0.0, + ) + partial_max = tl.load( + v_partial_ptr + partial_offset, + mask=mask[:, None], + other=0.0, + ) + sum_acc += tl.sum(partial_sum, axis=0) + max_acc = tl.maximum(max_acc, tl.max(partial_max, axis=0)) + chunk_base += REDUCE_TILE + + mean = sum_acc / seqlen_k + scale = tl.maximum(max_acc, 1.0e-7) / SCALE_MAX + tl.store( + k_mean_ptr + batch_idx * mean_stride_b + head_idx * mean_stride_h + dim_idx, + mean, + ) + tl.store( + v_scale_ptr + batch_idx * scale_stride_b + head_idx * scale_stride_h + dim_idx, + scale, + ) + + +@triton.jit +def _quantize_sage_kv_kernel( + k_ptr, + v_ptr, + k_mean_ptr, + v_scale_ptr, + k8_ptr, + v8_ptr, + k_scale_ptr, + k_stride_b, + k_stride_h, + k_stride_s, + v_stride_b, + v_stride_h, + v_stride_s, + mean_stride_b, + mean_stride_h, + v_scale_stride_b, + v_scale_stride_h, + k8_stride_b, + k8_stride_h, + k8_stride_s, + v8_stride_b, + v8_stride_h, + v8_stride_d, + k_scale_stride_b, + k_scale_stride_h, + seqlen_k, + batch_size, + HEAD_DIM: tl.constexpr, + BLOCK_SIZE: tl.constexpr, +): + block_idx = tl.program_id(0) + head_idx = tl.program_id(1) + batch_idx = tl.program_id(2) + local_row = tl.arange(0, BLOCK_SIZE) + row_idx = block_idx * BLOCK_SIZE + local_row + dim_idx = tl.arange(0, HEAD_DIM) + valid = row_idx < seqlen_k + + mean = tl.load( + k_mean_ptr + batch_idx * mean_stride_b + head_idx * mean_stride_h + dim_idx + ).to(tl.float32) + v_scale = tl.load( + v_scale_ptr + + batch_idx * v_scale_stride_b + + head_idx * v_scale_stride_h + + dim_idx + ).to(tl.float32) + k = tl.load( + k_ptr + + batch_idx * k_stride_b + + head_idx * k_stride_h + + row_idx[:, None] * k_stride_s + + dim_idx[None, :], + mask=valid[:, None], + other=0.0, + ).to(tl.float32) + v = tl.load( + v_ptr + + batch_idx * v_stride_b + + head_idx * v_stride_h + + row_idx[:, None] * v_stride_s + + dim_idx[None, :], + mask=valid[:, None], + other=0.0, + ).to(tl.float32) + + k_centered = tl.where(valid[:, None], k - mean[None, :], 0.0) + row_max = tl.max(tl.abs(k_centered), axis=1) + k_amax = tl.maximum(tl.max(row_max, axis=0), 1.0e-7) + k_scale = k_amax / 127.0 + k_quant = tl.maximum( + tl.minimum(libdevice.rint(k_centered / k_scale), 127.0), + -127.0, + ) + tl.store( + k8_ptr + + batch_idx * k8_stride_b + + head_idx * k8_stride_h + + row_idx[:, None] * k8_stride_s + + dim_idx[None, :], + k_quant.to(tl.int8), + mask=valid[:, None], + ) + tl.store( + k_scale_ptr + + batch_idx * k_scale_stride_b + + head_idx * k_scale_stride_h + + block_idx, + k_scale, + ) + + # This 16-token physical permutation must stay bit-for-bit consistent + # with the P-fragment byte-lane shuffle in + # flash_fwd_sm120_sage._make_acc_into_fp8_op -- the two together make the + # FP8 PV MMA's A-operand mapping lane-local without a runtime transpose. + row_mod = local_row % 16 + physical_row = ( + block_idx * BLOCK_SIZE + + (local_row // 16) * 16 + + (row_mod // 8) * 2 + + ((row_mod // 2) % 4) * 4 + + row_mod % 2 + ) + v_quant = tl.where(valid[:, None], v / v_scale[None, :], 0.0) + tl.store( + v8_ptr + + batch_idx * v8_stride_b + + head_idx * v8_stride_h + + dim_idx[:, None] * v8_stride_d + + physical_row[None, :], + tl.trans(v_quant).to(v8_ptr.dtype.element_ty), + ) + + +# ============================================================================= +# Public API +# ============================================================================= + + +def _require_sm120_bf16_bhsd(name: str, tensor: torch.Tensor) -> None: + if tensor.ndim != 4: + raise ValueError(f"{name} must be a rank-4 BHSD tensor") + if tensor.dtype != torch.bfloat16: + raise TypeError(f"{name} must use torch.bfloat16") + if not tensor.is_cuda: + raise ValueError(f"{name} must be a CUDA tensor") + if tensor.stride(-1) != 1: + raise ValueError(f"{name} must have a contiguous head-dim (stride(-1) == 1)") + if tensor.shape[-1] != SAGE_HEAD_DIM: + raise ValueError(f"{name} must have head dimension 128") + if tensor.shape[0] < 1 or tensor.shape[1] < 1 or tensor.shape[2] < 1: + raise ValueError(f"{name} requires positive B, H, and sequence length") + if torch.cuda.get_device_capability(tensor.device) != (12, 0): + raise RuntimeError("SM120 Sage quantization requires compute capability 12.0") + + +def _require_output( + name: str, + tensor: torch.Tensor, + shape: tuple, + dtype: torch.dtype, + device: torch.device, +) -> None: + if tensor.shape != shape: + raise ValueError(f"{name} must have shape {shape}, got {tuple(tensor.shape)}") + if tensor.dtype != dtype: + raise TypeError(f"{name} must use {dtype}") + if tensor.device != device: + raise ValueError(f"{name} must be on {device}") + if not tensor.is_contiguous(): + raise ValueError(f"{name} must be contiguous") + + +def _normalize_out( + out: Optional[Sequence[torch.Tensor]], + specs: tuple, + device: torch.device, +) -> tuple: + if out is None: + return tuple( + torch.empty(shape, dtype=dtype, device=device) for _, shape, dtype in specs + ) + if len(out) != len(specs): + raise ValueError(f"out must contain {len(specs)} tensors") + result = tuple(out) + for tensor, (name, shape, dtype) in zip(result, specs, strict=True): + _require_output(name, tensor, shape, dtype, device) + return result + + +def quantize_sage_q_sm120( + q: torch.Tensor, + *, + out: Optional[Sequence[torch.Tensor]] = None, +): + """Quantize BF16 BHSD Q to Sage per-32-token-group INT8 on SM120. + + Args: + q: Query tensor, contiguous BHSD bfloat16, head_dim=128. + out: Optional pre-allocated ``(q_int8, q_scale)`` output tensors. + + Returns: + ``(q_int8, q_scale)``: ``q_int8`` is int8 with the same BHSD shape as + ``q``. ``q_scale`` is float32 with shape + ``(B, H, ceil(Sq/128) * 4)`` -- one scale per 32-token group, padded + to four groups per 128-query tile. + """ + _require_sm120_bf16_bhsd("Q", q) + batch, heads, seqlen_q, head_dim = q.shape + num_groups = triton.cdiv(seqlen_q, SAGE_Q_BLOCK_SIZE) * ( + SAGE_Q_BLOCK_SIZE // SAGE_Q_GROUP_SIZE + ) + q_int8, q_scale = _normalize_out( + out, + ( + ("q_int8", tuple(q.shape), torch.int8), + ("q_scale", (batch, heads, num_groups), torch.float32), + ), + q.device, + ) + _quantize_sage_q_kernel[(num_groups, heads, batch)]( + q, + q_int8, + q_scale, + q.stride(0), + q.stride(1), + q.stride(2), + q_int8.stride(0), + q_int8.stride(1), + q_int8.stride(2), + q_scale.stride(0), + q_scale.stride(1), + seqlen_q, + batch, + HEAD_DIM=head_dim, + GROUP_SIZE=SAGE_Q_GROUP_SIZE, + num_warps=8, + ) + return q_int8, q_scale + + +def quantize_sage_kv_sm120( + k: torch.Tensor, + v: torch.Tensor, + *, + out: Optional[Sequence[torch.Tensor]] = None, +): + """Quantize BF16 BHSD K/V to Sage K64 INT8 and per-channel FP8 on SM120. + + K is channel-mean-centered before INT8 quantization (one scale per K64 + tile). V is FP8 E4M3 with a per-``[B,H,D]``-channel scale, stored in + Sage's ``[B, H, D, round_up(Sk, 64)]`` layout with the 16-token physical + permutation baked in by the kernel used to feed the FP8 PV MMA directly + (see flash_fwd_sm120_sage._make_acc_into_fp8_op). + + Args: + k: Key tensor, contiguous BHSD bfloat16, head_dim=128. + v: Value tensor, same shape/dtype/layout as ``k``. + out: Optional pre-allocated ``(k_int8, v_fp8, k_scale, v_scale)`` + output tensors. + + Returns: + ``(k_int8, v_fp8, k_scale, v_scale)``. + """ + _require_sm120_bf16_bhsd("K", k) + _require_sm120_bf16_bhsd("V", v) + if v.shape != k.shape: + raise ValueError("V must have the same shape as K") + if v.device != k.device: + raise ValueError("K and V must be on the same CUDA device") + + batch, heads, seqlen_k, head_dim = k.shape + padded_len = triton.cdiv(seqlen_k, SAGE_K_BLOCK_SIZE) * SAGE_K_BLOCK_SIZE + num_blocks = padded_len // SAGE_K_BLOCK_SIZE + specs = ( + ("k_int8", tuple(k.shape), torch.int8), + ("v_fp8", (batch, heads, head_dim, padded_len), torch.float8_e4m3fn), + ("k_scale", (batch, heads, num_blocks), torch.float32), + ("v_scale", (batch, heads, head_dim), torch.float32), + ) + k_int8, v_fp8, k_scale, v_scale = _normalize_out(out, specs, k.device) + + num_chunks = triton.cdiv(seqlen_k, SAGE_KV_STATS_CHUNK) + dim_tile = 32 + k_partial = torch.empty( + (batch, heads, num_chunks, head_dim), dtype=torch.float32, device=k.device + ) + v_partial = torch.empty_like(k_partial) + k_mean = torch.empty( + (batch, heads, head_dim), dtype=torch.bfloat16, device=k.device + ) + + _sage_kv_stats_partial_kernel[(num_chunks, head_dim // dim_tile, batch * heads)]( + k, + v, + k_partial, + v_partial, + k.stride(0), + k.stride(1), + k.stride(2), + v.stride(0), + v.stride(1), + v.stride(2), + k_partial.stride(0), + k_partial.stride(1), + k_partial.stride(2), + seqlen_k, + heads, + batch, + HEAD_DIM=head_dim, + CHUNK_SIZE=SAGE_KV_STATS_CHUNK, + DIM_TILE=dim_tile, + num_warps=8, + ) + _sage_kv_stats_finalize_kernel[(head_dim // dim_tile, batch * heads)]( + k_partial, + v_partial, + k_mean, + v_scale, + k_partial.stride(0), + k_partial.stride(1), + k_partial.stride(2), + k_mean.stride(0), + k_mean.stride(1), + v_scale.stride(0), + v_scale.stride(1), + num_chunks, + seqlen_k, + heads, + batch, + DIM_TILE=dim_tile, + REDUCE_TILE=16, + SCALE_MAX=SAGE_V_SCALE_MAX, + num_warps=4, + ) + _quantize_sage_kv_kernel[(num_blocks, heads, batch)]( + k, + v, + k_mean, + v_scale, + k_int8, + v_fp8, + k_scale, + k.stride(0), + k.stride(1), + k.stride(2), + v.stride(0), + v.stride(1), + v.stride(2), + k_mean.stride(0), + k_mean.stride(1), + v_scale.stride(0), + v_scale.stride(1), + k_int8.stride(0), + k_int8.stride(1), + k_int8.stride(2), + v_fp8.stride(0), + v_fp8.stride(1), + v_fp8.stride(2), + k_scale.stride(0), + k_scale.stride(1), + seqlen_k, + batch, + HEAD_DIM=head_dim, + BLOCK_SIZE=SAGE_K_BLOCK_SIZE, + num_warps=8, + ) + return k_int8, v_fp8, k_scale, v_scale + + +def quantize_sage_qkv_sm120( + q: torch.Tensor, + k: torch.Tensor, + v: torch.Tensor, + *, + out: Optional[Sequence[torch.Tensor]] = None, +): + """Quantize BF16 BHSD Q/K/V using the native Sage SM120 contract. + + Convenience wrapper around :func:`quantize_sage_q_sm120` and + :func:`quantize_sage_kv_sm120`. + + Returns: + ``(q_int8, k_int8, v_fp8, q_scale, k_scale, v_scale)``. + """ + if q.shape[:2] != k.shape[:2] or k.shape != v.shape: + raise ValueError("Q, K, and V must use matching batch/head dimensions") + if out is None: + q_out = None + kv_out = None + else: + if len(out) != 6: + raise ValueError("out must contain six tensors") + q_out = (out[0], out[3]) + kv_out = (out[1], out[2], out[4], out[5]) + + q_int8, q_scale = quantize_sage_q_sm120(q, out=q_out) + k_int8, v_fp8, k_scale, v_scale = quantize_sage_kv_sm120(k, v, out=kv_out) + return q_int8, k_int8, v_fp8, q_scale, k_scale, v_scale + + +__all__ = [ + "quantize_sage_kv_sm120", + "quantize_sage_q_sm120", + "quantize_sage_qkv_sm120", +] diff --git a/flashinfer/cute_dsl/sparse/sm120_blk64/flash_fwd_sm120_sage.py b/flashinfer/cute_dsl/sparse/sm120_blk64/flash_fwd_sm120_sage.py new file mode 100644 index 00000000000..8fe6a298a3e --- /dev/null +++ b/flashinfer/cute_dsl/sparse/sm120_blk64/flash_fwd_sm120_sage.py @@ -0,0 +1,1102 @@ +# Copyright (c) 2025 by FlashInfer team. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# Ported from Block-Sparse-Attention's csrc/fwd/sm120_blk64/bsa_fwd_sm120_sage.py +# (native SM120 Sage QK-INT8 / PV-FP8 block-sparse attention forward kernel). +# Structurally a sibling of flash_fwd_sm120.py: same TMA/pipeline/scheduler +# infrastructure, but QK runs through a custom INT8 warp MMA (MmaInt8Op, the +# SM80-era m16n8k32 s8s8s32 atom -- SM120 has no newer native INT8 tensor-core +# instruction) with an INT32 accumulator folded straight into a log2-domain +# online softmax, and PV runs through the native FP8 warp MMA. +# Does not compute LSE; exposed as a standalone function rather than through +# BlockSparseAttentionWrapper/_vsa_run_core. + +import math +import operator +from dataclasses import dataclass + +import cutlass +import cutlass.cute as cute +import cutlass.pipeline as pipeline +import cutlass.utils as utils +import cutlass.utils.hopper_helpers as sm90_utils +from cutlass.cutlass_dsl import T +from cutlass.cute.nvgpu.warp import mma as warp_mma +import cuda.bindings.driver as cuda + +from . import layout_utils +from . import kernel_utils +from .batched_static_scheduler import BatchedStaticSchedulerMixin +from .flash_fwd_sm120 import mask as _mask_fp8 + + +SM120_SAGE_FWD_BLOCK_SIZE = 64 +# Coupled to SAGE_V_SCALE_MAX (sage_quant_sm120.py): the FP16 PV accumulator +# overflows if FP8_MAX(448) * SAGE_V_SCALE_MAX * K_tile(64) >= 65504. +# Raising this scale or the K-tile size requires re-checking that bound. +SAGE_P_QUANT_SCALE = 256.0 +SAGE_P_QUANT_LOG2_SCALE = math.log2(SAGE_P_QUANT_SCALE) +SAGE_P_RESCALE_THRESHOLD = math.log2(448.0 / SAGE_P_QUANT_SCALE) + + +class MmaInt8Trait(warp_mma.Trait): + pass + + +@dataclass(frozen=True) +class MmaInt8Op(warp_mma.WarpMmaOp): + """Warp-level signed INT8 MMA with INT32 accumulation.""" + + ab_dtype: type[cutlass.Numeric] + acc_dtype: type[cutlass.Numeric] + shape_mnk: tuple[int, int, int] + + def __post_init__(self) -> None: + if self.ab_dtype is not cutlass.Int8: + raise TypeError("MmaInt8Op requires signed Int8 operands") + if self.acc_dtype is not cutlass.Int32: + raise TypeError("MmaInt8Op requires Int32 accumulation") + if self.shape_mnk != (16, 8, 32): + raise ValueError("MmaInt8Op requires the m16n8k32 instruction shape") + + def _make_trait(self, *, loc=None, ip=None, **kwargs): + shape_mnk = warp_mma._pack_shape(self.shape_mnk, loc=loc, ip=ip) + atom_type = warp_mma._cute_nvgpu_ir.MmaAtomSM80Type.get( + shape_mnk.type.attribute, + T.si8(), + T.si8(), + self.acc_dtype.mlir_type, + ) + return MmaInt8Trait(warp_mma.make_atom(atom_type, loc=loc, ip=ip)) + + def _verify_fragment_A(self, input, *, loc=None, ip=None) -> bool: + return True + + def _verify_fragment_B(self, input, *, loc=None, ip=None) -> bool: + return True + + +# ============================================================================= +# Public kernel class +# ============================================================================= +class BlockSparseAttnForwardSageSm120Blk64(BatchedStaticSchedulerMixin): + def __init__( + self, + gqa_ratio: int = 1, + head_dim: int = 128, + value_dim: int = 128, + blocksparse_blocksize_q: int = 64, + blocksparse_blocksize_k: int = 64, + has_block_sizes: bool = True, + has_block_nums: bool = True, + block_sizes_mode: int = 0, + ): + self.qk_acc_dtype = cutlass.Int32 + self.pv_acc_dtype = cutlass.Float16 + self.output_acc_dtype = cutlass.Float32 + self.softmax_p_scale_log2 = SAGE_P_QUANT_LOG2_SCALE + self.softmax_rescale_threshold = SAGE_P_RESCALE_THRESHOLD + + self.tile_size = SM120_SAGE_FWD_BLOCK_SIZE + + assert blocksparse_blocksize_q == self.tile_size, ( + "Only block_size_m=64 is supported in this kernel." + ) + assert blocksparse_blocksize_k == self.tile_size, ( + "Only block_size_n=64 is supported in this kernel." + ) + self.num_threads = 128 + # kv_stage is fixed at 1. Measured on SM120 (ncu, block_size=128): + # 230 registers/thread and 41KB dynamic smem/block (Q 8KB + K 8KB + + # V 8KB + O 16KB), both independently capping occupancy at 2 + # blocks/SM (65536 regs/SM -> floor(65536/(230*128))=2; + # 100KB smem/SM -> floor(100KB/41KB)=2). kv_stage=2 compiles fine + # (doubling K/V to 56KB/block is still far under the ~100KB/block + # opt-in max), but it drops the *smem* block limit to + # floor(100KB/56KB)=1 while leaving the register-derived limit at 2 + # (tSrK/tOrV stay single fragments regardless of depth), so smem + # becomes the new, tighter bottleneck -- occupancy would regress + # from 2 blocks/SM to 1. Whether the deeper pipeline's extra latency + # hiding offsets that occupancy loss is unmeasured (would need a + # runtime comparison); kv_stage=1 is kept as the higher-occupancy + # choice, not because it is proven the faster one. + # V's smem index is therefore always 0 (== K_consumer_state.index when + # kv_stage == 1), which is why it is hardcoded rather than tracked via + # V_consumer_state.index. + self.kv_stage = 1 + self.q_stage = 1 + + assert gqa_ratio >= 1 + assert head_dim == 128, "SM120 blk64 fwd currently requires QK dim 128" + assert value_dim == 128, "SM120 blk64 fwd currently requires value dim 128" + self.gqa_ratio = gqa_ratio + self.qk_dim = head_dim + self.value_dim = value_dim + self.tile_shape_qk = (self.tile_size, self.tile_size, self.qk_dim) + self.tile_shape_pv = (self.tile_size, self.value_dim, self.tile_size) + + self.has_block_sizes = has_block_sizes + self.has_block_nums = has_block_nums + self.block_sizes_mode = block_sizes_mode + + def _check_dim(self, tensor: cute.Tensor | list[cute.Tensor], mode: int): + if isinstance(tensor, list): + for t in tensor: + self._check_dim(t, mode) + return + assert tensor.shape[mode] == 128, f"dim must be 128 in mode {mode}." + assert tensor.stride[mode] == 1, f"dim must be contiguous in mode {mode}." + + @cute.kernel + def kernel( + self, + mQ: cute.Tensor, + mK: cute.Tensor, + mV: cute.Tensor, + mO: cute.Tensor, + mQScale: cute.Tensor, + mKScale: cute.Tensor, + mVScale: cute.Tensor, + tma_atom_Q: cute.CopyAtom, + tma_atom_K: cute.CopyAtom, + tma_atom_V: cute.CopyAtom, + tma_atom_O: cute.CopyAtom, + blocksparse_indices_q2k: cute.Tensor, + blocksparse_num_blocks_q2k: cute.Tensor, + block_sparse_num: cutlass.Int32, + blocksparse_varblk: cute.Tensor, + tiled_mma_qk: cute.TiledMma, + tiled_mma_pv: cute.TiledMma, + Q_smem_layout: cute.ComposedLayout, + K_smem_layout: cute.ComposedLayout, + V_smem_layout: cute.ComposedLayout, + O_smem_layout: cute.ComposedLayout, + scale_softmax_log2e: cutlass.Float32, + ): + tidx, _, _ = cute.arch.thread_idx() + lane_idx = cute.arch.lane_idx() + warp_idx = cute.arch.make_warp_uniform(cute.arch.warp_idx()) + work_desc = self.get_work_desc() + seqlen = mK.shape[0] + num_compute_tiles = cute.ceil_div(seqlen, self.tile_size) + + shared_storage = cutlass.utils.SmemAllocator().allocate(self.shared_storage_t) + + if warp_idx == 0 and lane_idx == 0: + cute.nvgpu.cpasync.prefetch_descriptor(tma_atom_Q) + cute.nvgpu.cpasync.prefetch_descriptor(tma_atom_K) + cute.nvgpu.cpasync.prefetch_descriptor(tma_atom_V) + cute.nvgpu.cpasync.prefetch_descriptor(tma_atom_O) + + cg = pipeline.CooperativeGroup(pipeline.Agent.Thread) + + # Q, K, and V use independent TMA pipelines. + Q_pipeline = pipeline.PipelineTmaAsync.create( + num_stages=self.q_stage, + producer_group=cg, + consumer_group=pipeline.CooperativeGroup( + pipeline.Agent.Thread, self.num_threads // 32 + ), + tx_count=cute.size_in_bytes( + self.Q_dtype, cute.select(Q_smem_layout, mode=[0, 1]) + ), + barrier_storage=shared_storage.Q_barrier.data_ptr(), + cta_layout_vmnk=cute.make_layout((1, 1, 1, 1)), + ) + Q_producer_state = pipeline.make_pipeline_state( + pipeline.PipelineUserType.Producer, self.q_stage + ) + Q_consumer_state = pipeline.make_pipeline_state( + pipeline.PipelineUserType.Consumer, self.q_stage + ) + K_pipeline = pipeline.PipelineTmaAsync.create( + num_stages=self.kv_stage, + producer_group=cg, + consumer_group=pipeline.CooperativeGroup( + pipeline.Agent.Thread, self.num_threads // 32 + ), + tx_count=cute.size_in_bytes( + self.K_dtype, cute.select(K_smem_layout, mode=[0, 1]) + ), + barrier_storage=shared_storage.K_barrier.data_ptr(), + cta_layout_vmnk=cute.make_layout((1, 1, 1, 1)), + ) + V_pipeline = pipeline.PipelineTmaAsync.create( + num_stages=self.kv_stage, + producer_group=cg, + consumer_group=pipeline.CooperativeGroup( + pipeline.Agent.Thread, self.num_threads // 32 + ), + tx_count=cute.size_in_bytes( + self.V_dtype, cute.select(V_smem_layout, mode=[0, 1]) + ), + barrier_storage=shared_storage.V_barrier.data_ptr(), + cta_layout_vmnk=cute.make_layout((1, 1, 1, 1)), + ) + K_producer_state = pipeline.make_pipeline_state( + pipeline.PipelineUserType.Producer, self.kv_stage + ) + K_consumer_state = pipeline.make_pipeline_state( + pipeline.PipelineUserType.Consumer, self.kv_stage + ) + V_producer_state = pipeline.make_pipeline_state( + pipeline.PipelineUserType.Producer, self.kv_stage + ) + V_consumer_state = pipeline.make_pipeline_state( + pipeline.PipelineUserType.Consumer, self.kv_stage + ) + + # Partition tensors. + sQ = shared_storage.Q_smem.get_tensor( + Q_smem_layout.outer, + swizzle=Q_smem_layout.inner, + ) + sK = shared_storage.K_smem.get_tensor( + K_smem_layout.outer, swizzle=K_smem_layout.inner + ) + sV = shared_storage.V_smem.get_tensor( + V_smem_layout.outer, swizzle=V_smem_layout.inner + ) + + mO_slice = mO[None, None, work_desc.qo_head_idx, work_desc.batch_idx] + mQ_slice = mQ[None, None, work_desc.qo_head_idx, work_desc.batch_idx] + mK_slice = mK[None, None, work_desc.kv_head_idx, work_desc.batch_idx] + mV_slice = mV[None, None, work_desc.kv_head_idx, work_desc.batch_idx] + gQScale = mQScale[None, work_desc.qo_head_idx, work_desc.batch_idx] + gKScale = mKScale[None, work_desc.kv_head_idx, work_desc.batch_idx] + gVScale = mVScale[None, work_desc.kv_head_idx, work_desc.batch_idx] + gO = cute.local_tile( + mO_slice, + (self.tile_shape_pv[0], self.tile_shape_pv[1]), + coord=(work_desc.qo_tile_idx, 0), + ) + gQ = cute.local_tile( + mQ_slice, + (self.tile_shape_qk[0], self.tile_shape_qk[2]), + coord=(work_desc.qo_tile_idx, 0), + ) + gK = cute.local_tile( + mK_slice, + (self.tile_shape_qk[1], self.tile_shape_qk[2]), + coord=(None, 0), + ) + gV = cute.local_tile( + mV_slice, + (self.tile_shape_pv[1], self.tile_shape_pv[2]), + coord=(0, None), + ) + + gIndices = blocksparse_indices_q2k[ + None, + work_desc.qo_tile_idx, + work_desc.qo_head_idx, + work_desc.batch_idx, + ] + if cutlass.const_expr(self.has_block_nums): + num_n_tiles = blocksparse_num_blocks_q2k[ + work_desc.qo_tile_idx, work_desc.qo_head_idx, work_desc.batch_idx + ] + else: + num_n_tiles = block_sparse_num + if cutlass.const_expr(self.has_block_sizes): + if cutlass.const_expr(self.block_sizes_mode == 1): + gBSZ = blocksparse_varblk + elif cutlass.const_expr(self.block_sizes_mode == 2): + gBSZ = blocksparse_varblk[None, work_desc.batch_idx] + else: + gBSZ = blocksparse_varblk[ + None, work_desc.qo_head_idx, work_desc.batch_idx + ] + + # A single-CTA layout disables TMA multicast. + cta_coord_layout = (0, cute.make_layout(1)) + tQsQ, tQgQ = cute.nvgpu.cpasync.tma_partition( + tma_atom_Q, + *cta_coord_layout, + cute.group_modes(sQ, 0, 2), + cute.group_modes(gQ, 0, 2), + ) + tKsK, tKgK = cute.nvgpu.cpasync.tma_partition( + tma_atom_K, + *cta_coord_layout, + cute.group_modes(sK, 0, 2), + cute.group_modes(gK, 0, 2), + ) + tVsV, tVgV = cute.nvgpu.cpasync.tma_partition( + tma_atom_V, + *cta_coord_layout, + cute.group_modes(sV, 0, 2), + cute.group_modes(gV, 0, 2), + ) + + cS = cute.make_identity_tensor(self.tile_shape_qk[:2]) + + thr_mma_qk = tiled_mma_qk.get_slice(tidx) + tSsQ = thr_mma_qk.partition_A(sQ) + tSsK = thr_mma_qk.partition_B(sK) + tSrQ = tiled_mma_qk.make_fragment_A(tSsQ[None, None, None, 0]) + tSrK = tiled_mma_qk.make_fragment_B(tSsK[None, None, None, 0]) + tSrS_i32 = cute.make_rmem_tensor( + thr_mma_qk.partition_shape_C( + (self.tile_shape_qk[0], self.tile_shape_qk[1]) + ), + self.qk_acc_dtype, + ) + tSrS = cute.make_rmem_tensor_like(tSrS_i32, cutlass.Float32) + tScS = thr_mma_qk.partition_C(cS) + + thr_mma_pv = tiled_mma_pv.get_slice(tidx) + tOsV = thr_mma_pv.partition_B(sV) + tOrV = tiled_mma_pv.make_fragment_B(tOsV[None, None, None, 0]) + tOrO = cute.make_rmem_tensor( + thr_mma_pv.partition_shape_C( + (self.tile_shape_pv[0], self.tile_shape_pv[1]) + ), + self.output_acc_dtype, + ) + tOrO_block = cute.make_rmem_tensor_like(tOrO, self.pv_acc_dtype) + cO = cute.make_identity_tensor(self.tile_shape_pv[:2]) + tOcO = thr_mma_pv.partition_C(cO) + + atom_copy_ldmatrix_Q = cute.make_copy_atom( + cute.nvgpu.warp.LdMatrix8x8x16bOp( + transpose=False, + num_matrices=4, + ), + self.Q_dtype, + ) + atom_copy_ldmatrix_K = cute.make_copy_atom( + cute.nvgpu.warp.LdMatrix8x8x16bOp( + transpose=False, + num_matrices=4, + ), + self.K_dtype, + ) + atom_copy_ldmatrix_V = cute.make_copy_atom( + cute.nvgpu.warp.LdMatrix8x8x16bOp( + transpose=False, + num_matrices=4, + ), + self.V_dtype, + ) + smem_tiled_copy_Q = cute.make_tiled_copy_A(atom_copy_ldmatrix_Q, tiled_mma_qk) + smem_tiled_copy_K = cute.make_tiled_copy_B(atom_copy_ldmatrix_K, tiled_mma_qk) + smem_tiled_copy_V = cute.make_tiled_copy_B( + atom_copy_ldmatrix_V, + tiled_mma_pv, + ) + + thr_copy_Q = smem_tiled_copy_Q.get_slice(tidx) + thr_copy_K = smem_tiled_copy_K.get_slice(tidx) + thr_copy_V = smem_tiled_copy_V.get_slice(tidx) + tSsQ_copy = thr_copy_Q.partition_S(sQ) + tSrQ_copy = thr_copy_Q.retile(tSrQ) + tSsK_copy = thr_copy_K.partition_S(sK) + tOsV_copy = thr_copy_V.partition_S(sV) + + max_m_layout = cute.make_layout( + cute.size( + layout_utils.reshape_acc_to_mn(tOrO).layout, + mode=[0], + ) + ) + max_m = cute.make_rmem_tensor_like(max_m_layout, cutlass.Float32) + sum_m = cute.make_rmem_tensor_like(max_m, cutlass.Float32) + q_softmax_scale_log2e_m = cute.make_rmem_tensor_like(max_m, cutlass.Float32) + tScS_mn = layout_utils.reshape_acc_to_mn(tScS) + for m in cutlass.range_constexpr(cute.size(q_softmax_scale_log2e_m)): + row_idx = work_desc.qo_tile_idx * self.tile_size + tScS_mn[m, 0][0] + q_scale_idx = (row_idx // 128) * 4 + (row_idx % 128) // 32 + q_softmax_scale_log2e_m[m] = ( + gQScale[q_scale_idx] * scale_softmax_log2e + if row_idx < mQ.shape[0] + else cutlass.Float32(0.0) + ) + + tOrO.store(cute.full_like(tOrO, 0.0, self.output_acc_dtype)) + max_m.store(cute.full_like(max_m, float("-inf"), cutlass.Float32)) + sum_m.store(cute.full_like(sum_m, 0.0, cutlass.Float32)) + + if warp_idx == 0: + Q_pipeline.producer_acquire(Q_producer_state) + cute.copy( + tma_atom_Q, + tQgQ, + tQsQ[None, 0], + tma_bar_ptr=Q_pipeline.producer_get_barrier(Q_producer_state), + ) + Q_pipeline.producer_commit(Q_producer_state) + Q_producer_state.advance() + + preload_count = cutlass.Int32(0) + if preload_count < num_n_tiles: + logical_idx = num_n_tiles - 1 - preload_count + physical_idx = gIndices[logical_idx] + K_pipeline.producer_acquire(K_producer_state) + cute.copy( + tma_atom_K, + tKgK[None, physical_idx], + tKsK[None, K_producer_state.index], + tma_bar_ptr=K_pipeline.producer_get_barrier(K_producer_state), + ) + K_pipeline.producer_commit(K_producer_state) + K_producer_state.advance() + V_pipeline.producer_acquire(V_producer_state) + cute.copy( + tma_atom_V, + tVgV[None, physical_idx], + tVsV[None, V_producer_state.index], + tma_bar_ptr=V_pipeline.producer_get_barrier(V_producer_state), + ) + V_pipeline.producer_commit(V_producer_state) + V_producer_state.advance() + cute.arch.sync_threads() + + Q_wait_status = Q_pipeline.consumer_try_wait(Q_consumer_state) + Q_pipeline.consumer_wait(Q_consumer_state, Q_wait_status) + tQsQ_p = tSsQ_copy[None, None, None, 0] + for k_block_idx in cutlass.range_constexpr(cute.size(tSrQ, mode=[2])): + tQsQ_k = tQsQ_p[None, None, k_block_idx] + tQsQ_k = cute.make_tensor( + tQsQ_k.iterator.align(16), + tQsQ_k.layout, + ) + cute.copy( + smem_tiled_copy_Q, + tQsQ_k, + tSrQ_copy[None, None, k_block_idx], + ) + Q_pipeline.consumer_release(Q_consumer_state) + Q_consumer_state.advance() + + n_tile_idx = cutlass.Int32(0) + if num_n_tiles > 0: + n_tile_idx = gIndices[num_n_tiles - 1] + for load_count in cutlass.range(0, num_n_tiles, 1, unroll=1): + if cutlass.const_expr(self.has_block_sizes): + varblk = gBSZ[n_tile_idx] + else: + varblk = cutlass.Int32(self.tile_size) + if n_tile_idx == num_compute_tiles - 1: + varblk = seqlen - n_tile_idx * self.tile_size + + K_wait_status = K_pipeline.consumer_try_wait(K_consumer_state) + K_pipeline.consumer_wait(K_consumer_state, K_wait_status) + + k_stage = K_consumer_state.index + + _gemm_qk_int8( + tiled_mma_qk, + tSrS_i32, + tSrQ, + tSrK, + tSsK_copy[None, None, None, k_stage], + smem_tiled_copy_K, + ) + + tSrS.store(tSrS_i32.load().to(cutlass.Float32)) + + k_scale = _load_sage_k_scale_int8( + gKScale, + n_tile_idx, + ) + + K_pipeline.consumer_release(K_consumer_state) + K_consumer_state.advance() + + preload_count = load_count + self.kv_stage + next_n_tile_idx = cutlass.Int32(0) + if preload_count < num_n_tiles: + preload_logical = num_n_tiles - 1 - preload_count + next_n_tile_idx = gIndices[preload_logical] + if warp_idx == 0: + K_pipeline.producer_acquire(K_producer_state) + cute.copy( + tma_atom_K, + tKgK[None, next_n_tile_idx], + tKsK[None, K_producer_state.index], + tma_bar_ptr=K_pipeline.producer_get_barrier(K_producer_state), + ) + K_pipeline.producer_commit(K_producer_state) + K_producer_state.advance() + + if varblk < self.tile_size: + _mask_fp8(tSrS, tScS, varblk) + row_scale = _online_softmax_fp8( + tSrS, + max_m, + sum_m, + q_softmax_scale_log2e_m, + self.softmax_p_scale_log2, + self.softmax_rescale_threshold, + k_scale, + ) + + # Compute P @ V. + _rescale_o_for_next_acc_fp8(tOrO, row_scale) + tOrP = _make_acc_into_fp8_op( + tSrS, + tiled_mma_pv.tv_layout_A, + self.V_dtype, + ) + + V_wait_status = V_pipeline.consumer_try_wait(V_consumer_state) + V_pipeline.consumer_wait(V_consumer_state, V_wait_status) + + tOrO_block.fill(0.0) + _gemm_rs_fp8( + tiled_mma_pv, + tOrO_block, + tOrP, + tOrV, + tOsV_copy[None, None, None, 0], + smem_tiled_copy_V, + ) + _accumulate_o_block_fp8(tOrO, tOrO_block) + V_pipeline.consumer_release(V_consumer_state) + V_consumer_state.advance() + + if warp_idx == 0 and preload_count < num_n_tiles: + V_pipeline.producer_acquire(V_producer_state) + cute.copy( + tma_atom_V, + tVgV[None, next_n_tile_idx], + tVsV[None, V_producer_state.index], + tma_bar_ptr=V_pipeline.producer_get_barrier(V_producer_state), + ) + V_pipeline.producer_commit(V_producer_state) + V_producer_state.advance() + + n_tile_idx = next_n_tile_idx + + final_ratio = _finalize_softmax_sage(sum_m) + _rescale_o_with_sage_v_scale_fp8( + tOrO, + tOcO, + final_ratio, + gVScale, + ) + tOrO_cvt = cute.make_rmem_tensor_like(tOrO, self.O_dtype) + tOrO_cvt.store(tOrO.load().to(self.O_dtype)) + + # Use a separate shared-memory buffer for the BF16 output tile. + sO = shared_storage.O_smem.get_tensor( + O_smem_layout.outer, swizzle=O_smem_layout.inner + ) + tiled_copy_o_r2s = cute.make_tiled_copy_C( + cute.make_copy_atom( + cute.nvgpu.warp.StMatrix8x8x16bOp(self.O_layout.is_m_major_c(), 4), + self.O_dtype, + ), + tiled_mma_pv, + ) + tOrO_cv = tiled_copy_o_r2s.retile(tOrO_cvt) + tOsO = tiled_copy_o_r2s.get_slice(tidx).partition_D(sO) + cute.copy(tiled_copy_o_r2s, tOrO_cv, tOsO) + + cute.arch.fence_view_async_shared() + cute.arch.sync_threads() + + # S2G with explicit TMA store wait. + tOsO, tOgO = cute.nvgpu.cpasync.tma_partition( + tma_atom_O, + *cta_coord_layout, + cute.group_modes(sO, 0, 2), + cute.group_modes(gO, 0, 2), + ) + if warp_idx == 0: + cute.copy(tma_atom_O, tOsO, tOgO) + cute.arch.cp_async_bulk_commit_group() + cute.arch.cp_async_bulk_wait_group(0, read=True) + + @cute.jit + def __call__( + self, + mQ: cute.Tensor, # (head_dim, seqlen, nheads, batch) + mK: cute.Tensor, # (head_dim, seqlen, nheads, batch) + mV: cute.Tensor, # (value_dim, seqlen, nheads, batch) + mO: cute.Tensor, # (value_dim, seqlen, nheads, batch) + mQScale: cute.Tensor, # (ceil_div(seqlen_q, 128) * 4, nheads_q, batch) + mKScale: cute.Tensor, # (ceil_div(seqlen_k, 64), nheads_kv, batch) + mVScale: cute.Tensor, # (value_dim, nheads_kv, batch) + blocksparse_indices_q2k: cute.Tensor, # (k, q, nheads, batch) + blocksparse_num_blocks_q2k: cute.Tensor, # (q, nheads, batch) + block_sparse_num: cutlass.Int32, + blocksparse_varblk: cute.Tensor, + softmax_scale: cutlass.Float32, + stream: cuda.CUstream, + ): + # Restore compile-time head dimensions while keeping runtime tensor + # modes dynamic for AOT and reusable JIT artifacts. + mQ = cute.make_tensor( + mQ.iterator, + cute.make_layout( + (mQ.shape[0], self.qk_dim, mQ.shape[2], mQ.shape[3]), + stride=mQ.stride, + ), + ) + mK = cute.make_tensor( + mK.iterator, + cute.make_layout( + (mK.shape[0], self.qk_dim, mK.shape[2], mK.shape[3]), + stride=mK.stride, + ), + ) + mV = cute.make_tensor( + mV.iterator, + cute.make_layout( + (self.value_dim, mV.shape[1], mV.shape[2], mV.shape[3]), + stride=mV.stride, + ), + ) + mO = cute.make_tensor( + mO.iterator, + cute.make_layout( + (mO.shape[0], self.value_dim, mO.shape[2], mO.shape[3]), + stride=mO.stride, + ), + ) + mVScale = cute.make_tensor( + mVScale.iterator, + cute.make_layout( + (self.value_dim, mVScale.shape[1], mVScale.shape[2]), + stride=mVScale.stride, + ), + ) + self._check_dim([mQ, mK, mO], 1) + assert mV.shape[0] == self.value_dim + + Q_layout = utils.LayoutEnum.from_tensor(mQ) + K_layout = utils.LayoutEnum.from_tensor(mK) + V_layout = utils.LayoutEnum.from_tensor(mV) + O_layout = utils.LayoutEnum.from_tensor(mO) + + self.Q_dtype = mQ.element_type + self.K_dtype = mK.element_type + self.V_dtype = mV.element_type + self.O_dtype = mO.element_type + assert self.Q_dtype is cutlass.Int8 + assert self.K_dtype is cutlass.Int8 + assert self.V_dtype is cutlass.Float8E4M3FN + assert self.O_dtype is cutlass.BFloat16 + self.Q_layout = Q_layout + self.K_layout = K_layout + self.V_layout = V_layout + self.O_layout = O_layout + + atom_layout_mnk = (4, 1, 1) + mma_inst_mnk = (16, 8, 32) + permutation_mnk = ( + atom_layout_mnk[0] * mma_inst_mnk[0], + atom_layout_mnk[1] * mma_inst_mnk[1] * 2, + atom_layout_mnk[2] * mma_inst_mnk[2], + ) + mma_op_qk = MmaInt8Op( + self.Q_dtype, + self.qk_acc_dtype, + mma_inst_mnk, + ) + mma_op_pv = cute.nvgpu.warp.MmaFP8Op( + self.V_dtype, + self.pv_acc_dtype, + mma_inst_mnk, + ) + tiled_mma_qk = cute.make_tiled_mma( + mma_op_qk, + cute.make_layout(atom_layout_mnk), + permutation_mnk=permutation_mnk, + ) + tiled_mma_pv = cute.make_tiled_mma( + mma_op_pv, + cute.make_layout(atom_layout_mnk), + permutation_mnk=permutation_mnk, + ) + + self.Q_smem_layout = sm90_utils.make_smem_layout_a( + Q_layout, + self.tile_shape_qk, + self.Q_dtype, + self.q_stage, + ) + self.K_smem_layout = sm90_utils.make_smem_layout_b( + K_layout, + self.tile_shape_qk, + self.K_dtype, + self.kv_stage, + ) + self.V_smem_layout = sm90_utils.make_smem_layout_b( + V_layout, + self.tile_shape_pv, + self.V_dtype, + self.kv_stage, + ) + O_smem_layout_staged = sm90_utils.make_smem_layout_epi( + self.O_dtype, + O_layout, + self.tile_shape_pv[:2], + 1, + ) + self.O_smem_layout = cute.select(O_smem_layout_staged, mode=[0, 1]) + + @cute.struct + class SharedStorage: + Q_barrier: cute.struct.MemRange[cutlass.Int64, self.q_stage * 2] + K_barrier: cute.struct.MemRange[cutlass.Int64, self.kv_stage * 2] + V_barrier: cute.struct.MemRange[cutlass.Int64, self.kv_stage * 2] + + Q_smem: cute.struct.Align[ + cute.struct.MemRange[self.Q_dtype, cute.cosize(self.Q_smem_layout)], + 128, + ] + K_smem: cute.struct.Align[ + cute.struct.MemRange[self.K_dtype, cute.cosize(self.K_smem_layout)], + 128, + ] + V_smem: cute.struct.Align[ + cute.struct.MemRange[self.V_dtype, cute.cosize(self.V_smem_layout)], + 128, + ] + O_smem: cute.struct.Align[ + cute.struct.MemRange[self.O_dtype, cute.cosize(self.O_smem_layout)], + 128, + ] + + self.shared_storage_t = SharedStorage + + tma_copy_op = cute.nvgpu.cpasync.CopyBulkTensorTileG2SOp() + tma_atom_Q, tma_tensor_Q = cute.nvgpu.cpasync.make_tiled_tma_atom( + tma_copy_op, + mQ, + self.Q_smem_layout, + (self.tile_shape_qk[0], self.tile_shape_qk[2]), + num_multicast=1, + ) + tma_atom_K, tma_tensor_K = cute.nvgpu.cpasync.make_tiled_tma_atom( + tma_copy_op, + mK, + self.K_smem_layout, + (self.tile_shape_qk[1], self.tile_shape_qk[2]), + num_multicast=1, + ) + tma_atom_V, tma_tensor_V = cute.nvgpu.cpasync.make_tiled_tma_atom( + tma_copy_op, + mV, + self.V_smem_layout, + (self.tile_shape_pv[1], self.tile_shape_pv[2]), + num_multicast=1, + ) + + tma_copy_op = cute.nvgpu.cpasync.CopyBulkTensorTileS2GOp() + tma_atom_O, tma_tensor_O = cute.nvgpu.cpasync.make_tiled_tma_atom( + tma_copy_op, + mO, + self.O_smem_layout, + (self.tile_shape_pv[0], self.tile_shape_pv[1]), + num_multicast=1, + ) + + log2_e = 1.44269504088896340736 + + grid_config = self.get_grid_config(mQ.shape[0], mQ.shape[2], mQ.shape[3]) + block_config = (self.num_threads, 1, 1) + + self.kernel( + tma_tensor_Q, + tma_tensor_K, + tma_tensor_V, + tma_tensor_O, + mQScale, + mKScale, + mVScale, + tma_atom_Q, + tma_atom_K, + tma_atom_V, + tma_atom_O, + blocksparse_indices_q2k, + blocksparse_num_blocks_q2k, + block_sparse_num, + blocksparse_varblk, + tiled_mma_qk, + tiled_mma_pv, + self.Q_smem_layout, + self.K_smem_layout, + self.V_smem_layout, + self.O_smem_layout, + softmax_scale * log2_e, + ).launch( + grid=grid_config, + block=block_config, + cluster=(1, 1, 1), + smem=self.shared_storage_t.size_in_bytes(), # type: ignore[attr-defined] + stream=stream, + min_blocks_per_mp=1, + ) + + +# ============================================================================= +# Local CuTe helpers +# ============================================================================= + + +def _convert_c_layout_to_a_layout_fp8(c: cute.Layout, a: cute.Layout) -> cute.Layout: + """Reinterpret a C fragment with the logical shape of an A operand.""" + a_layout = cute.make_layout(a) + c_atom_size = cute.size(c, mode=[0]) + expansion = cute.size(a) // c_atom_size + return cute.make_layout( + (a, c.shape[1], c.shape[2] // expansion), + stride=( + a_layout.stride, + 0, + cute.size(a), + ), + ) + + +@cute.jit +def _make_acc_into_fp8_op( + acc: cute.Tensor, + operand_layout_tv: cute.Layout, + element: type[cutlass.Numeric], +) -> cute.Tensor: + """Convert the score accumulator directly into Sage's FP8 A fragment. + + NOTE: the byte-lane permutation below (via cute.arch.prmt) is the + inverse-side counterpart of the 16-token physical permutation baked into + V by the quantization kernel (see sage_quant_sm120.py, + _quantize_sage_kv_kernel's `physical_row` computation). The two must stay + bit-for-bit consistent -- porting or editing one without the other + silently produces wrong (not crashing) numerical results. + """ + operand = cute.make_rmem_tensor_like( + _convert_c_layout_to_a_layout_fp8(acc.layout, operand_layout_tv.shape[1]), + element, + ) + operand_as_acc = cute.make_tensor(operand.iterator, acc.layout) + operand_as_acc.store(acc.load().to(element)) + + # Composing the standard C-to-A exchange with Sage's 16-token V + # permutation makes the final mapping lane-local. Pack the low and high + # byte pairs from adjacent registers exactly like Sage's RS_32_to_8. + values_u32 = cute.recast_tensor(operand, cutlass.Uint32) + for n in cutlass.range_constexpr(cute.size(values_u32, mode=[1])): + for k in cutlass.range_constexpr(cute.size(values_u32, mode=[2])): + for ii in cutlass.range_constexpr(0, 8, 4): + values_tmp_0 = values_u32[ii // 2, n, k] + values_tmp_1 = values_u32[ii // 2 + 1, n, k] + values_u32[ii // 2, n, k] = cute.arch.prmt( + values_tmp_0, values_tmp_1, 0x5410 + ) + values_u32[ii // 2 + 1, n, k] = cute.arch.prmt( + values_tmp_0, values_tmp_1, 0x7632 + ) + return operand + + +@cute.jit +def _gemm_qk_int8( + tiled_mma: cute.TiledMma, + acc: cute.Tensor, + tCrA: cute.Tensor, + tCrB: cute.Tensor, + tCsB: cute.Tensor, + smem_tiled_copy_B: cute.TiledCopy, +) -> None: + """Run INT8 QK MMA with Q in registers and K staged in shared memory.""" + acc.fill(0.0) + tCrB_copy_view = smem_tiled_copy_B.retile(tCrB) + tCsB_cur = tCsB[None, None, 0] + tCsB_cur = cute.make_tensor( + tCsB_cur.iterator.align(16), + tCsB_cur.layout, + ) + cute.copy(smem_tiled_copy_B, tCsB_cur, tCrB_copy_view[None, None, 0]) + for k_block_idx in cutlass.range_constexpr(cute.size(tCsB.shape[2])): + if k_block_idx < cute.size(tCsB.shape[2]) - 1: + tCsB_next = tCsB[None, None, k_block_idx + 1] + tCsB_next = cute.make_tensor( + tCsB_next.iterator.align(16), + tCsB_next.layout, + ) + cute.copy( + smem_tiled_copy_B, + tCsB_next, + tCrB_copy_view[None, None, k_block_idx + 1], + ) + cute.gemm( + tiled_mma, + acc, + tCrA[None, None, k_block_idx], + tCrB[None, None, k_block_idx], + acc, + ) + + +@cute.jit +def _gemm_rs_fp8( + tiled_mma: cute.TiledMma, + acc: cute.Tensor, + tCrA: cute.Tensor, + tCrB: cute.Tensor, + tCsB: cute.Tensor, + smem_tiled_copy_B: cute.TiledCopy, +) -> None: + """Run FP8 PV MMA with V staged in shared memory.""" + tCrB_copy_view = smem_tiled_copy_B.retile(tCrB) + tCsB_cur = tCsB[None, None, 0] + cute.copy(smem_tiled_copy_B, tCsB_cur, tCrB_copy_view[None, None, 0]) + for k_block_idx in cutlass.range_constexpr(cute.size(tCrA.shape[2])): + if k_block_idx < cute.size(tCrA.shape[2]) - 1: + tCsB_next = tCsB[None, None, k_block_idx + 1] + cute.copy( + smem_tiled_copy_B, + tCsB_next, + tCrB_copy_view[None, None, k_block_idx + 1], + ) + cute.gemm( + tiled_mma, + acc, + tCrA[None, None, k_block_idx], + tCrB[None, None, k_block_idx], + acc, + ) + + +@cute.jit +def _load_sage_k_scale_int8( + gKScale: cute.Tensor, + k_tile_idx: cutlass.Int32, +) -> cutlass.Float32: + """Load the single Sage K64 descale for one physical KV tile.""" + scale_idx = k_tile_idx if k_tile_idx < gKScale.shape[0] else gKScale.shape[0] - 1 + return gKScale[scale_idx] + + +@cute.jit +def _online_softmax_fp8( + tSrS: cute.ThrMma, + row_max: cute.Tensor, + row_sum: cute.Tensor, + softmax_scale_log2e_m: cute.Tensor, + exp_scale_log2: cutlass.Float32, + rescale_threshold: cutlass.Constexpr[float], + k_scale: cutlass.Float32, +) -> cute.Tensor: + tSrS_mn = layout_utils.reshape_acc_to_mn(tSrS) + row_scale = cute.make_rmem_tensor_like(row_max, cutlass.Float32) + + for m in cutlass.range(cute.size(row_max), unroll_full=True): + score_scale_log2e = k_scale * softmax_scale_log2e_m[m] + row_max_local = row_max[m] + # Keep row_max in the final exp2 domain. Sage has one positive K scale + # per K64 tile, so the unscaled scores can be reduced before scaling. + group_max = tSrS_mn[m, 0] + for n in cutlass.range_constexpr( + 1, + cute.size(tSrS_mn, mode=[1]), + ): + group_max = cute.arch.fmax(group_max, tSrS_mn[m, n]) + row_max_local = cute.arch.fmax( + row_max_local, + group_max * score_scale_log2e, + ) + row_max_cur = cute.arch.warp_reduction_max( + row_max_local, + threads_in_group=4, + ) + + row_max_prev = row_max[m] + row_max_safe = ( + cutlass.Float32(0.0) if row_max_cur == -cutlass.Float32.inf else row_max_cur + ) + row_scale_log2 = row_max_prev - row_max_safe + if row_scale_log2 >= -rescale_threshold: + row_max_cur = row_max_prev + row_max_safe = row_max_prev + row_scale[m] = 1.0 + else: + row_scale[m] = cute.math.exp2( + row_scale_log2, + fastmath=True, + ) + row_max[m] = row_max_cur + # Shift the exponent by log2(P scale) before FP8 conversion. Keeping + # row_sum in the same scale removes a vector multiply from every tile. + row_max_shifted = row_max_safe - exp_scale_log2 + for n in cutlass.range_constexpr(cute.size(tSrS_mn, mode=[1])): + tSrS_mn[m, n] = cute.math.exp2( + tSrS_mn[m, n] * score_scale_log2e - row_max_shifted, + fastmath=True, + ) + acc_S_row_exp = tSrS_mn[m, None].load() + row_sum[m] = kernel_utils.fadd_reduce( + acc_S_row_exp, + init_val=row_sum[m] * row_scale[m], + arch=80, + ) + tSrS_mn[m, None].store(acc_S_row_exp) + + return row_scale + + +@cute.jit +def _finalize_softmax_sage(row_sum: cute.Tensor) -> cute.Tensor: + """Reduce the scaled denominator and return its reciprocal.""" + row_sum.store(kernel_utils.warp_reduce(row_sum.load(), operator.add, width=4)) + final_ratio = cute.make_rmem_tensor_like(row_sum, cutlass.Float32) + + for m in cutlass.range(cute.size(row_sum), unroll_full=True): + final_sum = row_sum[m] + is_zero_or_nan = final_sum == 0.0 or final_sum != final_sum + final_ratio[m] = cute.arch.rcp_approx(final_sum if not is_zero_or_nan else 1.0) + + return final_ratio + + +@cute.jit +def _rescale_o_for_next_acc_fp8( + tOrO: cute.ThrMma, + prev_ratio_m: cute.Tensor, +): + tOrO_mn = layout_utils.reshape_acc_to_mn(tOrO) + for m in cutlass.range(cute.size(prev_ratio_m), unroll_full=True): + should_rescale = cute.arch.vote_ballot_sync(prev_ratio_m[m] < 1.0) != 0 + if should_rescale: + tOrO_mn[m, None].store(tOrO_mn[m, None].load() * prev_ratio_m[m]) + + +@cute.jit +def _accumulate_o_block_fp8( + tOrO: cute.Tensor, + tOrO_block: cute.Tensor, +) -> None: + """Promote each FP16 PV result and merge it into the FP32 O accumulator.""" + for i in cutlass.range(cute.size(tOrO), unroll_full=True): + tOrO[i] += tOrO_block[i].to(cutlass.Float32) + + +@cute.jit +def _rescale_o_with_sage_v_scale_fp8( + tOrO: cute.Tensor, + tOcO: cute.Tensor, + final_ratio_m: cute.Tensor, + gVScale: cute.Tensor, +) -> None: + """Normalize O and apply the per-channel Sage V descale.""" + tOrO_mn = layout_utils.reshape_acc_to_mn(tOrO) + tOcO_mn = layout_utils.reshape_acc_to_mn(tOcO) + for n in cutlass.range(cute.size(tOrO_mn, mode=[1]), unroll_full=True): + v_scale = gVScale[tOcO_mn[0, n][1]] + for m in cutlass.range(cute.size(final_ratio_m), unroll_full=True): + tOrO_mn[m, n] *= final_ratio_m[m] * v_scale diff --git a/tests/attention/test_vsa_block_sparse_sm120_sage.py b/tests/attention/test_vsa_block_sparse_sm120_sage.py new file mode 100644 index 00000000000..4dc74b6ece1 --- /dev/null +++ b/tests/attention/test_vsa_block_sparse_sm120_sage.py @@ -0,0 +1,978 @@ +""" +Copyright (c) 2025 by FlashInfer team. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" + +import math +from unittest import mock + +import pytest +import torch + +from flashinfer.cute_dsl.sparse.bsa_attn_sm120 import bsa_attn_sm120_blk64_sage_fwd +from flashinfer.cute_dsl.sparse.bsa_utils.sage_quant_sm120 import ( + quantize_sage_kv_sm120, + quantize_sage_q_sm120, + quantize_sage_qkv_sm120, +) +# --------------------------------------------------------------------------- +# Hardware gate +# --------------------------------------------------------------------------- + +pytestmark = pytest.mark.skipif( + not torch.cuda.is_available() or torch.cuda.get_device_capability() != (12, 0), + reason="sm120_blk64 Sage backend requires SM120 (compute capability 12.0)", +) + +BLOCK = 64 +HEAD_DIM = 128 +# Sage QK-INT8/PV-FP8 quantization is inherently lossy; use a looser +# tolerance than the bf16/fp16 vsa_sm120_blk64 tests (atol/rtol=1e-2). +ATOL, RTOL = 5e-2, 5e-2 + + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + + +def _build_random_q2k( + batch: int, + heads: int, + num_q_blocks: int, + num_kv_blocks: int, + density: float, + device: torch.device, +): + """Return (q2k_block_index, q2k_block_nums) with a variable KV-block count + per (batch, head, q_block) row; every row has >= 1 block.""" + capacity = num_kv_blocks + index = torch.zeros( + batch, heads, num_q_blocks, capacity, dtype=torch.int32, device=device + ) + nums = torch.zeros(batch, heads, num_q_blocks, dtype=torch.int32, device=device) + for b in range(batch): + for h in range(heads): + for qi in range(num_q_blocks): + k = max(1, int(round(density * num_kv_blocks))) + k = min(k, num_kv_blocks) + chosen = torch.randperm(num_kv_blocks)[:k].sort().values + index[b, h, qi, :k] = chosen.to(torch.int32).to(device) + nums[b, h, qi] = k + return index, nums + + +def _dense_mask_from_q2k( + q2k_index: torch.Tensor, + q2k_nums: torch.Tensor, + seqlen_q: int, + seqlen_k: int, + device: torch.device, +) -> torch.Tensor: + """Expand direct-selection q2k metadata into a per-(batch,head) token mask + [B, H, seqlen_q, seqlen_k].""" + batch, heads, num_q_blocks, _capacity = q2k_index.shape + mask = torch.zeros( + batch, heads, seqlen_q, seqlen_k, dtype=torch.bool, device=device + ) + idx_cpu = q2k_index.cpu() + nums_cpu = q2k_nums.cpu() + for b in range(batch): + for h in range(heads): + for qi in range(num_q_blocks): + q_lo = qi * BLOCK + q_hi = min(q_lo + BLOCK, seqlen_q) + cnt = int(nums_cpu[b, h, qi]) + for j in range(cnt): + kb = int(idx_cpu[b, h, qi, j]) + k_lo = kb * BLOCK + k_hi = min(k_lo + BLOCK, seqlen_k) + mask[b, h, q_lo:q_hi, k_lo:k_hi] = True + return mask + + +def _pytorch_ref( + q: torch.Tensor, # [B, H, Sq, D] bf16 (pre-quantization) + k: torch.Tensor, # [B, H, Sk, D] bf16 + v: torch.Tensor, # [B, H, Sk, D] bf16 + mask: torch.Tensor, # [B, H, Sq, Sk] bool + sm_scale: float | None = None, +) -> torch.Tensor: + """Dense FP32 PyTorch reference computed on the pre-quantization tensors. + + This is the correctness bar for a lossy INT8/FP8 kernel: the quantized + kernel output should be close to (not bit-exact with) the full-precision + reference. + """ + D = q.shape[-1] + if sm_scale is None: + sm_scale = 1.0 / math.sqrt(D) + qf, kf, vf = q.float(), k.float(), v.float() + scores = torch.einsum("bhsd,bhtd->bhst", qf, kf) * sm_scale + scores = scores.masked_fill(~mask, float("-inf")) + probs = torch.softmax(scores, dim=-1) + return torch.einsum("bhst,bhtd->bhsd", probs, vf).to(torch.bfloat16) + + +def _random_bf16_bhsd(batch, heads, seqlen, device, scale=0.1): + return ( + torch.randn(batch, heads, seqlen, HEAD_DIM, dtype=torch.bfloat16, device=device) + * scale + ) + + +# --------------------------------------------------------------------------- +# Accuracy tests +# --------------------------------------------------------------------------- + + +@pytest.mark.parametrize( + "batch,num_heads,num_blocks,density", + [ + (1, 2, 2, 1.0), + (1, 8, 8, 0.5), + (2, 4, 4, 0.25), + (2, 8, 16, 0.75), + ], +) +def test_vsa_sm120_sage_accuracy(batch, num_heads, num_blocks, density): + device = torch.device("cuda") + torch.manual_seed(42) + seqlen = num_blocks * BLOCK + + q = _random_bf16_bhsd(batch, num_heads, seqlen, device) + k = _random_bf16_bhsd(batch, num_heads, seqlen, device) + v = _random_bf16_bhsd(batch, num_heads, seqlen, device) + + q8, k8, v8, qs, ks, vs = quantize_sage_qkv_sm120(q, k, v) + + q2k_index, q2k_nums = _build_random_q2k( + batch, num_heads, num_blocks, num_blocks, density, device + ) + mask = _dense_mask_from_q2k(q2k_index, q2k_nums, seqlen, seqlen, device) + o_ref = _pytorch_ref(q, k, v, mask) + + o = bsa_attn_sm120_blk64_sage_fwd( + q8, + k8, + v8, + qs, + ks, + vs, + q2k_index, + block_sparse_num=num_blocks, + q2k_block_nums=q2k_nums, + backend="cute_dsl", + ) + + torch.testing.assert_close(o.float(), o_ref.float(), atol=ATOL, rtol=RTOL) + + +@pytest.mark.parametrize("sm_scale", [0.5, 1.0 / math.sqrt(HEAD_DIM)]) +def test_vsa_sm120_sage_sm_scale(sm_scale): + device = torch.device("cuda") + torch.manual_seed(4) + batch, num_heads, num_blocks = 1, 4, 4 + seqlen = num_blocks * BLOCK + + q = _random_bf16_bhsd(batch, num_heads, seqlen, device) + k = _random_bf16_bhsd(batch, num_heads, seqlen, device) + v = _random_bf16_bhsd(batch, num_heads, seqlen, device) + q8, k8, v8, qs, ks, vs = quantize_sage_qkv_sm120(q, k, v) + + q2k_index, q2k_nums = _build_random_q2k( + batch, num_heads, num_blocks, num_blocks, 0.5, device + ) + mask = _dense_mask_from_q2k(q2k_index, q2k_nums, seqlen, seqlen, device) + o_ref = _pytorch_ref(q, k, v, mask, sm_scale=sm_scale) + + o = bsa_attn_sm120_blk64_sage_fwd( + q8, + k8, + v8, + qs, + ks, + vs, + q2k_index, + block_sparse_num=num_blocks, + q2k_block_nums=q2k_nums, + softmax_scale=sm_scale, + backend="cute_dsl", + ) + + torch.testing.assert_close(o.float(), o_ref.float(), atol=ATOL, rtol=RTOL) + + +@pytest.mark.parametrize( + "seqlen_q,seqlen_k", + [ + (100, 128), # non-64-aligned Q tail + (128, 100), # non-64-aligned K tail + (100, 100), # both tails + ], +) +def test_vsa_sm120_sage_ragged_seqlen(seqlen_q, seqlen_k): + """Non-64-aligned seqlens exercise the per-tile tail-masking path.""" + device = torch.device("cuda") + torch.manual_seed(13) + batch, num_heads = 1, 4 + num_q_blocks = (seqlen_q + BLOCK - 1) // BLOCK + num_kv_blocks = (seqlen_k + BLOCK - 1) // BLOCK + + q = _random_bf16_bhsd(batch, num_heads, seqlen_q, device) + k = _random_bf16_bhsd(batch, num_heads, seqlen_k, device) + v = _random_bf16_bhsd(batch, num_heads, seqlen_k, device) + q8, k8, v8, qs, ks, vs = quantize_sage_qkv_sm120(q, k, v) + + # Every Q block attends to every KV block (dense, but seqlen is ragged). + index = ( + torch.arange(num_kv_blocks, dtype=torch.int32, device=device) + .view(1, 1, 1, num_kv_blocks) + .expand(batch, num_heads, num_q_blocks, num_kv_blocks) + .contiguous() + ) + mask = torch.zeros( + batch, num_heads, seqlen_q, seqlen_k, dtype=torch.bool, device=device + ) + mask[:] = True + + o_ref = _pytorch_ref(q, k, v, mask) + o = bsa_attn_sm120_blk64_sage_fwd( + q8, + k8, + v8, + qs, + ks, + vs, + index, + block_sparse_num=num_kv_blocks, + backend="cute_dsl", + ) + + torch.testing.assert_close(o.float(), o_ref.float(), atol=ATOL, rtol=RTOL) + + +def test_vsa_sm120_sage_empty_row(): + """A Q-block with zero selected KV blocks must produce exactly zero output.""" + device = torch.device("cuda") + torch.manual_seed(42) + batch, num_heads, num_blocks = 1, 4, 4 + seqlen = num_blocks * BLOCK + + q = _random_bf16_bhsd(batch, num_heads, seqlen, device) + k = _random_bf16_bhsd(batch, num_heads, seqlen, device) + v = _random_bf16_bhsd(batch, num_heads, seqlen, device) + q8, k8, v8, qs, ks, vs = quantize_sage_qkv_sm120(q, k, v) + + index = torch.zeros( + batch, num_heads, num_blocks, num_blocks, dtype=torch.int32, device=device + ) + nums = torch.full( + (batch, num_heads, num_blocks), num_blocks, dtype=torch.int32, device=device + ) + for kb in range(num_blocks): + index[:, :, 1, kb] = kb + nums[:, :, 0] = 0 # Q-block 0 selects nothing. + + o = bsa_attn_sm120_blk64_sage_fwd( + q8, + k8, + v8, + qs, + ks, + vs, + index, + block_sparse_num=num_blocks, + q2k_block_nums=nums, + backend="cute_dsl", + ) + + assert torch.all(o[:, :, :BLOCK, :] == 0), "empty Q-block output should be zero" + assert torch.isfinite(o[:, :, BLOCK:, :]).all() + + +# --------------------------------------------------------------------------- +# API surface / argument plumbing +# --------------------------------------------------------------------------- + + +def test_vsa_sm120_sage_out_param(): + device = torch.device("cuda") + torch.manual_seed(2) + batch, num_heads, num_blocks = 1, 2, 2 + seqlen = num_blocks * BLOCK + + q = _random_bf16_bhsd(batch, num_heads, seqlen, device) + k = _random_bf16_bhsd(batch, num_heads, seqlen, device) + v = _random_bf16_bhsd(batch, num_heads, seqlen, device) + q8, k8, v8, qs, ks, vs = quantize_sage_qkv_sm120(q, k, v) + + index = ( + torch.tensor([[0, 1]], dtype=torch.int32, device=device) + .expand(batch, num_heads, num_blocks, num_blocks) + .contiguous() + ) + preallocated = torch.empty( + batch, num_heads, seqlen, HEAD_DIM, dtype=torch.bfloat16, device=device + ) + out = bsa_attn_sm120_blk64_sage_fwd( + q8, + k8, + v8, + qs, + ks, + vs, + index, + block_sparse_num=num_blocks, + out=preallocated, + backend="cute_dsl", + ) + assert out.data_ptr() == preallocated.data_ptr() + assert torch.isfinite(out).all() + + +@pytest.mark.parametrize("block_sizes_mode", [1, 2, 3]) +def test_vsa_sm120_sage_block_sizes(block_sizes_mode): + """block_sizes (1D / 2D / 3D, matching _prepare_sm120_sparse_metadata's + block_sizes_mode encoding) must mask out padding tokens in the last KV + block.""" + device = torch.device("cuda") + torch.manual_seed(7) + batch, num_heads = 1, 2 + num_blocks = 2 + seqlen = num_blocks * BLOCK + valid_last_block = 40 # last KV block only has 40/64 valid tokens + + q = _random_bf16_bhsd(batch, num_heads, seqlen, device) + k = _random_bf16_bhsd(batch, num_heads, seqlen, device) + v = _random_bf16_bhsd(batch, num_heads, seqlen, device) + q8, k8, v8, qs, ks, vs = quantize_sage_qkv_sm120(q, k, v) + + index = ( + torch.tensor([[0, 1]], dtype=torch.int32, device=device) + .expand(batch, num_heads, num_blocks, num_blocks) + .contiguous() + ) + + if block_sizes_mode == 1: + block_sizes = torch.tensor( + [BLOCK, valid_last_block], dtype=torch.int32, device=device + ) + elif block_sizes_mode == 2: + block_sizes = ( + torch.tensor([[BLOCK, valid_last_block]], dtype=torch.int32, device=device) + .expand(batch, num_blocks) + .contiguous() + ) + else: + block_sizes = ( + torch.tensor([[BLOCK, valid_last_block]], dtype=torch.int32, device=device) + .view(1, 1, num_blocks) + .expand(batch, num_heads, num_blocks) + .contiguous() + ) + + mask = torch.zeros( + batch, num_heads, seqlen, seqlen, dtype=torch.bool, device=device + ) + mask[:] = True + mask[:, :, :, BLOCK + valid_last_block :] = False + + o_ref = _pytorch_ref(q, k, v, mask) + o = bsa_attn_sm120_blk64_sage_fwd( + q8, + k8, + v8, + qs, + ks, + vs, + index, + block_sparse_num=num_blocks, + block_sizes=block_sizes, + backend="cute_dsl", + ) + + torch.testing.assert_close(o.float(), o_ref.float(), atol=ATOL, rtol=RTOL) + + +def test_vsa_sm120_sage_gqa_guard(): + """This function only supports MHA; num_kv_heads != num_heads must raise.""" + device = torch.device("cuda") + torch.manual_seed(0) + batch, num_heads, num_blocks = 1, 4, 2 + seqlen = num_blocks * BLOCK + + q = _random_bf16_bhsd(batch, num_heads, seqlen, device) + k = _random_bf16_bhsd(batch, 2, seqlen, device) + v = _random_bf16_bhsd(batch, 2, seqlen, device) + q8, qs = quantize_sage_q_sm120(q) + k8, v8, ks, vs = quantize_sage_kv_sm120(k, v) + + index = ( + torch.tensor([[0, 1]], dtype=torch.int32, device=device) + .expand(batch, num_heads, num_blocks, num_blocks) + .contiguous() + ) + + with pytest.raises(ValueError): + bsa_attn_sm120_blk64_sage_fwd( + q8, + k8, + v8, + qs, + ks, + vs, + index, + block_sparse_num=num_blocks, + backend="cute_dsl", + ) + + +def test_vsa_sm120_sage_wrong_arch_dtype_guards(): + """Dtype guards must reject non-int8 Q/K and non-fp8 V.""" + device = torch.device("cuda") + torch.manual_seed(0) + batch, num_heads, num_blocks = 1, 2, 2 + seqlen = num_blocks * BLOCK + index = ( + torch.tensor([[0, 1]], dtype=torch.int32, device=device) + .expand(batch, num_heads, num_blocks, num_blocks) + .contiguous() + ) + + q_bf16 = _random_bf16_bhsd(batch, num_heads, seqlen, device) + k = torch.zeros(batch, num_heads, seqlen, HEAD_DIM, dtype=torch.int8, device=device) + v = torch.zeros( + batch, num_heads, HEAD_DIM, seqlen, dtype=torch.float8_e4m3fn, device=device + ) + qs = torch.ones(batch, num_heads, 4, dtype=torch.float32, device=device) + ks = torch.ones(batch, num_heads, num_blocks, dtype=torch.float32, device=device) + vs = torch.ones(batch, num_heads, HEAD_DIM, dtype=torch.float32, device=device) + + with pytest.raises(TypeError): + bsa_attn_sm120_blk64_sage_fwd( + q_bf16, + k, + v, + qs, + ks, + vs, + index, + block_sparse_num=num_blocks, + backend="cute_dsl", + ) + + +# --------------------------------------------------------------------------- +# V permutation / PRMT consistency +# +# The V quantization kernel (_quantize_sage_kv_kernel in sage_quant_sm120.py) +# physically stores V under a 16-token permutation so the FP8 PV MMA can +# consume it without an in-kernel transpose. The forward kernel +# (flash_fwd_sm120_sage._make_acc_into_fp8_op) undoes this with a matching +# cute.arch.prmt byte-lane shuffle on the P fragment. If the two disagree, +# attention would still numerically "work" (finite, plausible-looking +# output) but silently retrieve the WRONG token's V row -- ordinary +# end-to-end accuracy tests with random data cannot distinguish this from a +# correct kernel, because averaging over many random tokens hides which +# specific token contributed what. This test is position-sensitive: query +# row t is engineered to attend almost exclusively to K/V token t, so a +# permutation mismatch would surface as row t recovering some *other* +# token's V value instead of its own. +# --------------------------------------------------------------------------- + + +def test_vsa_sm120_sage_v_permutation_consistency(): + """Query row t must recover V token t's value, exercising every token + position (and hence every branch of the 16-token physical permutation + formula) in a single kernel launch.""" + device = torch.device("cuda") + torch.manual_seed(0) + batch, num_heads = 1, 2 + S = BLOCK # single K64 tile: sweep all 64 possible physical-permutation slots + + # Diagonal-dominant Q/K: row t has a large spike in its own feature + # dimension t (plus small background noise), so Q[t]-K[j] is maximized + # at j == t for every row simultaneously. + bg, spike = 0.05, 10.0 + qf = torch.randn(batch, num_heads, S, HEAD_DIM, device=device) * bg + kf = torch.randn(batch, num_heads, S, HEAD_DIM, device=device) * bg + diag = torch.arange(S, device=device) + qf[:, :, diag, diag] = spike + kf[:, :, diag, diag] = spike + q = qf.to(torch.bfloat16) + k = kf.to(torch.bfloat16) + + # Every token gets a unique, monotonically increasing "fingerprint" + # value, identical across all 128 channels. + const = 0.01 + token_ids = torch.arange(S, device=device).float() + 1.0 + v = ( + (token_ids.view(1, 1, S, 1) * const) + .expand(batch, num_heads, S, HEAD_DIM) + .to(torch.bfloat16) + ) + + q8, k8, v8, qs, ks, vs = quantize_sage_qkv_sm120(q, k, v) + + index = torch.zeros(batch, num_heads, 1, 1, dtype=torch.int32, device=device) + # Large softmax_scale sharpens the (already diagonal-dominant) score + # matrix into a near-one-hot distribution. + out = bsa_attn_sm120_blk64_sage_fwd( + q8, + k8, + v8, + qs, + ks, + vs, + index, + block_sparse_num=1, + softmax_scale=30.0, + backend="cute_dsl", + ) + out_mean = out.float().mean(dim=-1) # [batch, num_heads, S] + + # The correctness bar is the FP8-rounded V value (V itself is quantized + # to float8_e4m3fn, ~6% per-octave precision), not the pre-quantization + # float value -- comparing against the latter would conflate V's + # inherent quantization error with an actual permutation bug. + v_scale_bc = vs.view(batch, num_heads, 1, HEAD_DIM) + v_fp8_sim = (v.float() / v_scale_bc).to(torch.float8_e4m3fn).float() + expected = (v_fp8_sim * v_scale_bc).mean(dim=-1) # [batch, num_heads, S] + + # Tight tolerance: correct behavior gives ~1e-3 residual (softmax not + # perfectly one-hot); a real permutation bug would misroute row t to a + # *different* token's fingerprint, which is spaced at least `const` + # apart -- an order of magnitude larger than this bound. + torch.testing.assert_close(out_mean, expected, atol=3e-3, rtol=0.0) + + +# --------------------------------------------------------------------------- +# Quantization unit tests +# --------------------------------------------------------------------------- + + +def test_quantize_sage_q_shapes_and_range(): + device = torch.device("cuda") + torch.manual_seed(0) + batch, heads, seqlen = 2, 3, 150 # not a multiple of 128 -> partial last group + q = _random_bf16_bhsd(batch, heads, seqlen, device, scale=1.0) + + q8, q_scale = quantize_sage_q_sm120(q) + assert q8.shape == q.shape + assert q8.dtype == torch.int8 + num_groups = ((seqlen + 127) // 128) * 4 + assert q_scale.shape == (batch, heads, num_groups) + assert q8.abs().max() <= 127 + + # Dequantized value should be close to the original (per-32-token-group scale). + group_of_row = (torch.arange(seqlen, device=device) // 32).clamp(max=num_groups - 1) + scale_per_row = q_scale[:, :, group_of_row] # [B, H, S] + dequant = q8.float() * scale_per_row.unsqueeze(-1) + torch.testing.assert_close(dequant, q.float(), atol=5e-2, rtol=5e-2) + + +def test_quantize_sage_kv_shapes_and_v_scale_cap(): + device = torch.device("cuda") + torch.manual_seed(1) + batch, heads, seqlen = 1, 2, 96 # not a multiple of 64 + k = _random_bf16_bhsd(batch, heads, seqlen, device, scale=1.0) + v = _random_bf16_bhsd(batch, heads, seqlen, device, scale=1.0) + + k8, v8, k_scale, v_scale = quantize_sage_kv_sm120(k, v) + padded = ((seqlen + BLOCK - 1) // BLOCK) * BLOCK + assert k8.shape == k.shape + assert k8.dtype == torch.int8 + assert v8.shape == (batch, heads, HEAD_DIM, padded) + assert v8.dtype == torch.float8_e4m3fn + assert k_scale.shape == (batch, heads, padded // BLOCK) + assert v_scale.shape == (batch, heads, HEAD_DIM) + # SAGE_V_SCALE_MAX cap: scale = max(amax, eps) / 2.25, so amax/scale should + # not exceed 2.25 (up to fp32 rounding). + assert (v.float().abs().amax(dim=2) / v_scale <= 2.25 + 1e-3).all() + + +@pytest.mark.parametrize( + "bad_tensor_name,mutate", + [ + ("q_scale", lambda t: t[:, :, :1].contiguous()), + ("k_scale", lambda t: t[:, :, :1].contiguous()), + ("v_scale", lambda t: t[:, :, :1].contiguous()), + ("k_int8", lambda t: t[:, :1].contiguous()), # wrong head count (not MHA) + ], +) +def test_vsa_sm120_sage_rejects_mismatched_shapes(bad_tensor_name, mutate): + """Wrong-shaped Q/K scale tensors or a non-MHA K must raise, not silently + compute a wrong result; an under-sized q_scale previously ran to completion + and returned a finite but numerically wrong output.""" + device = torch.device("cuda") + torch.manual_seed(0) + batch, num_heads, num_blocks = 1, 2, 2 + seqlen = num_blocks * BLOCK + + q = _random_bf16_bhsd(batch, num_heads, seqlen, device) + k = _random_bf16_bhsd(batch, num_heads, seqlen, device) + v = _random_bf16_bhsd(batch, num_heads, seqlen, device) + q8, k8, v8, qs, ks, vs = quantize_sage_qkv_sm120(q, k, v) + + tensors = { + "q_int8": q8, + "k_int8": k8, + "v_fp8": v8, + "q_scale": qs, + "k_scale": ks, + "v_scale": vs, + } + tensors[bad_tensor_name] = mutate(tensors[bad_tensor_name]) + + index = ( + torch.tensor([[0, 1]], dtype=torch.int32, device=device) + .expand(batch, num_heads, num_blocks, num_blocks) + .contiguous() + ) + + with pytest.raises(ValueError): + bsa_attn_sm120_blk64_sage_fwd( + tensors["q_int8"], + tensors["k_int8"], + tensors["v_fp8"], + tensors["q_scale"], + tensors["k_scale"], + tensors["v_scale"], + index, + block_sparse_num=num_blocks, + backend="cute_dsl", + ) + + +def test_cake_backend_requires_out(): + """backend='cake' must raise ValueError when out= is omitted.""" + device = torch.device("cuda") + torch.manual_seed(0) + batch, num_heads, num_blocks = 1, 2, 2 + seqlen = num_blocks * BLOCK + q = _random_bf16_bhsd(batch, num_heads, seqlen, device) + k = _random_bf16_bhsd(batch, num_heads, seqlen, device) + v = _random_bf16_bhsd(batch, num_heads, seqlen, device) + q8, k8, v8, qs, ks, vs = quantize_sage_qkv_sm120(q, k, v) + index = ( + torch.tensor([[0, 1]], dtype=torch.int32, device=device) + .expand(batch, num_heads, num_blocks, num_blocks) + .contiguous() + ) + workspace = torch.empty(1024 * 1024, dtype=torch.uint8, device=device) + with pytest.raises(ValueError, match="requires a caller-owned out tensor"): + bsa_attn_sm120_blk64_sage_fwd( + q8, + k8, + v8, + qs, + ks, + vs, + index, + block_sparse_num=num_blocks, + tma_descriptor_workspace=workspace, + backend="cake", + ) + + +def test_cake_backend_requires_tma_workspace(): + """backend='cake' must raise ValueError when tma_descriptor_workspace= is omitted.""" + device = torch.device("cuda") + torch.manual_seed(0) + batch, num_heads, num_blocks = 1, 2, 2 + seqlen = num_blocks * BLOCK + q = _random_bf16_bhsd(batch, num_heads, seqlen, device) + k = _random_bf16_bhsd(batch, num_heads, seqlen, device) + v = _random_bf16_bhsd(batch, num_heads, seqlen, device) + q8, k8, v8, qs, ks, vs = quantize_sage_qkv_sm120(q, k, v) + index = ( + torch.tensor([[0, 1]], dtype=torch.int32, device=device) + .expand(batch, num_heads, num_blocks, num_blocks) + .contiguous() + ) + out = torch.empty( + batch, num_heads, seqlen, HEAD_DIM, dtype=torch.bfloat16, device=device + ) + with pytest.raises(ValueError, match="requires tma_descriptor_workspace"): + bsa_attn_sm120_blk64_sage_fwd( + q8, + k8, + v8, + qs, + ks, + vs, + index, + block_sparse_num=num_blocks, + out=out, + backend="cake", + ) + + +def test_unsupported_backend(): + """An unknown backend name must raise ValueError.""" + device = torch.device("cuda") + torch.manual_seed(0) + batch, num_heads, num_blocks = 1, 2, 2 + seqlen = num_blocks * BLOCK + q = _random_bf16_bhsd(batch, num_heads, seqlen, device) + k = _random_bf16_bhsd(batch, num_heads, seqlen, device) + v = _random_bf16_bhsd(batch, num_heads, seqlen, device) + q8, k8, v8, qs, ks, vs = quantize_sage_qkv_sm120(q, k, v) + index = ( + torch.tensor([[0, 1]], dtype=torch.int32, device=device) + .expand(batch, num_heads, num_blocks, num_blocks) + .contiguous() + ) + with pytest.raises(ValueError, match="unsupported SM120 Sage backend"): + bsa_attn_sm120_blk64_sage_fwd( + q8, + k8, + v8, + qs, + ks, + vs, + index, + block_sparse_num=num_blocks, + backend="unknown", + ) + + +def test_vsa_sm120_sage_default_backend_is_cake(): + """The default `backend` must remain "cake": omitting it (and the + cake-required `out`/`tma_descriptor_workspace` kwargs) must raise the + cake-specific error, not silently succeed via cute_dsl (which allocates + `out` internally and does not require `tma_descriptor_workspace`).""" + device = torch.device("cuda") + torch.manual_seed(0) + batch, num_heads, num_blocks = 1, 2, 2 + seqlen = num_blocks * BLOCK + q = _random_bf16_bhsd(batch, num_heads, seqlen, device) + k = _random_bf16_bhsd(batch, num_heads, seqlen, device) + v = _random_bf16_bhsd(batch, num_heads, seqlen, device) + q8, k8, v8, qs, ks, vs = quantize_sage_qkv_sm120(q, k, v) + index = ( + torch.tensor([[0, 1]], dtype=torch.int32, device=device) + .expand(batch, num_heads, num_blocks, num_blocks) + .contiguous() + ) + with pytest.raises(ValueError, match="requires a caller-owned out tensor"): + bsa_attn_sm120_blk64_sage_fwd( + q8, + k8, + v8, + qs, + ks, + vs, + index, + block_sparse_num=num_blocks, + ) + + +# --------------------------------------------------------------------------- +# Additional validation coverage: block_sparse_num=0 without q2k_block_nums, +# and the guards added in the "SM120 Sage" review round (softmax_scale +# finiteness/positivity, out device/contiguity, inputs on the current CUDA +# device). +# --------------------------------------------------------------------------- + + +def test_vsa_sm120_sage_block_sparse_num_zero_scalar(): + """block_sparse_num=0 with no q2k_block_nums (the has_block_nums=False + branch, unlike test_vsa_sm120_sage_empty_row which goes through + q2k_block_nums) must select zero KV blocks for every Q row and produce + an exactly zero output.""" + device = torch.device("cuda") + torch.manual_seed(0) + batch, num_heads, num_blocks = 1, 2, 2 + seqlen = num_blocks * BLOCK + + q = _random_bf16_bhsd(batch, num_heads, seqlen, device) + k = _random_bf16_bhsd(batch, num_heads, seqlen, device) + v = _random_bf16_bhsd(batch, num_heads, seqlen, device) + q8, k8, v8, qs, ks, vs = quantize_sage_qkv_sm120(q, k, v) + + index = torch.zeros( + batch, num_heads, num_blocks, num_blocks, dtype=torch.int32, device=device + ) + + o = bsa_attn_sm120_blk64_sage_fwd( + q8, + k8, + v8, + qs, + ks, + vs, + index, + block_sparse_num=0, + backend="cute_dsl", + ) + + assert torch.all(o == 0), "block_sparse_num=0 must produce an all-zero output" + + +@pytest.mark.parametrize("bad_scale", [-1.0, 0.0, float("nan"), float("inf")]) +def test_vsa_sm120_sage_softmax_scale_guard(bad_scale): + """softmax_scale must be finite and strictly positive; negative, zero, + and NaN/inf must all raise ValueError rather than run the kernel. + ``-inf`` is not parametrized separately: it hits the same + ``math.isfinite()`` branch as ``inf`` and adds no new coverage.""" + device = torch.device("cuda") + torch.manual_seed(0) + batch, num_heads, num_blocks = 1, 2, 2 + seqlen = num_blocks * BLOCK + + q = _random_bf16_bhsd(batch, num_heads, seqlen, device) + k = _random_bf16_bhsd(batch, num_heads, seqlen, device) + v = _random_bf16_bhsd(batch, num_heads, seqlen, device) + q8, k8, v8, qs, ks, vs = quantize_sage_qkv_sm120(q, k, v) + index = ( + torch.tensor([[0, 1]], dtype=torch.int32, device=device) + .expand(batch, num_heads, num_blocks, num_blocks) + .contiguous() + ) + + with pytest.raises(ValueError): + bsa_attn_sm120_blk64_sage_fwd( + q8, + k8, + v8, + qs, + ks, + vs, + index, + block_sparse_num=num_blocks, + softmax_scale=bad_scale, + backend="cute_dsl", + ) + + +def test_vsa_sm120_sage_out_wrong_device_guard(): + """A pre-allocated `out` on a different device than the inputs must + raise ValueError instead of silently computing a cross-device result. + Uses a CPU `out` rather than a second CUDA device: the check being + tested is a plain ``out.device != q_int8.device`` comparison, so a CPU + tensor exercises it identically without requiring >=2 GPUs.""" + device = torch.device("cuda") + torch.manual_seed(0) + batch, num_heads, num_blocks = 1, 2, 2 + seqlen = num_blocks * BLOCK + + q = _random_bf16_bhsd(batch, num_heads, seqlen, device) + k = _random_bf16_bhsd(batch, num_heads, seqlen, device) + v = _random_bf16_bhsd(batch, num_heads, seqlen, device) + q8, k8, v8, qs, ks, vs = quantize_sage_qkv_sm120(q, k, v) + index = ( + torch.tensor([[0, 1]], dtype=torch.int32, device=device) + .expand(batch, num_heads, num_blocks, num_blocks) + .contiguous() + ) + out = torch.empty( + batch, num_heads, seqlen, HEAD_DIM, dtype=torch.bfloat16, device="cpu" + ) + + with pytest.raises(ValueError, match="out must be on"): + bsa_attn_sm120_blk64_sage_fwd( + q8, + k8, + v8, + qs, + ks, + vs, + index, + block_sparse_num=num_blocks, + out=out, + backend="cute_dsl", + ) + + +def test_vsa_sm120_sage_out_non_contiguous_guard(): + """A non-contiguous `out` must raise ValueError instead of writing + through a mismatched stride.""" + device = torch.device("cuda") + torch.manual_seed(0) + batch, num_heads, num_blocks = 1, 2, 2 + seqlen = num_blocks * BLOCK + + q = _random_bf16_bhsd(batch, num_heads, seqlen, device) + k = _random_bf16_bhsd(batch, num_heads, seqlen, device) + v = _random_bf16_bhsd(batch, num_heads, seqlen, device) + q8, k8, v8, qs, ks, vs = quantize_sage_qkv_sm120(q, k, v) + index = ( + torch.tensor([[0, 1]], dtype=torch.int32, device=device) + .expand(batch, num_heads, num_blocks, num_blocks) + .contiguous() + ) + # Same logical shape/dtype as a valid `out`, but transposed to a + # non-contiguous view. + out = torch.empty( + batch, num_heads, HEAD_DIM, seqlen, dtype=torch.bfloat16, device=device + ).transpose(2, 3) + assert out.shape == (batch, num_heads, seqlen, HEAD_DIM) + assert not out.is_contiguous() + + with pytest.raises(ValueError, match="out must be contiguous"): + bsa_attn_sm120_blk64_sage_fwd( + q8, + k8, + v8, + qs, + ks, + vs, + index, + block_sparse_num=num_blocks, + out=out, + backend="cute_dsl", + ) + + +def test_vsa_sm120_sage_inputs_not_on_current_device_guard(): + """Inputs living on a CUDA device other than `torch.cuda.current_device()` + must raise ValueError instead of launching against the wrong context. + The check being tested is a plain + ``q_int8.device.index != torch.cuda.current_device()`` comparison, so + this mocks ``torch.cuda.current_device()`` to report a device index + that differs from where the (real, single-GPU) inputs live, rather + than requiring >=2 physical GPUs.""" + device = torch.device("cuda") + torch.manual_seed(0) + batch, num_heads, num_blocks = 1, 2, 2 + seqlen = num_blocks * BLOCK + + q = _random_bf16_bhsd(batch, num_heads, seqlen, device) + k = _random_bf16_bhsd(batch, num_heads, seqlen, device) + v = _random_bf16_bhsd(batch, num_heads, seqlen, device) + q8, k8, v8, qs, ks, vs = quantize_sage_qkv_sm120(q, k, v) + index = ( + torch.tensor([[0, 1]], dtype=torch.int32, device=device) + .expand(batch, num_heads, num_blocks, num_blocks) + .contiguous() + ) + + fake_current_device = q8.device.index + 1 + with ( + mock.patch("torch.cuda.current_device", return_value=fake_current_device), + pytest.raises(ValueError, match="current CUDA device"), + ): + bsa_attn_sm120_blk64_sage_fwd( + q8, + k8, + v8, + qs, + ks, + vs, + index, + block_sparse_num=num_blocks, + backend="cute_dsl", + )