Skip to content
3 changes: 2 additions & 1 deletion csrc/batch_mla_binding.cu
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,8 @@ void BatchMLAPagedAttentionRun(TensorView float_workspace_buffer, TensorView int
TensorView ckv_cache, TensorView kpe_cache, TensorView kv_indices,
TensorView o, Optional<TensorView> maybe_lse, int64_t mask_mode_code,
int64_t num_heads, int64_t page_size, double sm_scale,
bool return_lse_base_on_e, double ckv_scale, double kpe_scale);
bool return_lse_base_on_e, double ckv_scale, double kpe_scale,
Optional<TensorView> maybe_ckv_scale_arr);

TVM_FFI_DLL_EXPORT_TYPED_FUNC(plan, BatchMLAPagedAttentionPlan);
TVM_FFI_DLL_EXPORT_TYPED_FUNC(run, BatchMLAPagedAttentionRun);
7 changes: 6 additions & 1 deletion csrc/batch_mla_run.cu
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,8 @@ void BatchMLAPagedAttentionRun(TensorView float_workspace_buffer, TensorView int
TensorView ckv_cache, TensorView kpe_cache, TensorView kv_indices,
TensorView o, Optional<TensorView> maybe_lse, int64_t mask_mode_code,
int64_t num_heads, int64_t page_size, double sm_scale,
bool return_lse_base_on_e, double ckv_scale, double kpe_scale) {
bool return_lse_base_on_e, double ckv_scale, double kpe_scale,
Optional<TensorView> maybe_ckv_scale_arr) {
// q_nope: [n, num_heads, head_dim_ckv]
// q_pe: [n, num_heads, head_dim_kpe]
// ckv_cache: [num_pages, page_size, head_dim_ckv]
Expand Down Expand Up @@ -117,6 +118,10 @@ void BatchMLAPagedAttentionRun(TensorView float_workspace_buffer, TensorView int
params.sm_scale = sm_scale;
params.ckv_scale = static_cast<float>(ckv_scale);
params.kpe_scale = static_cast<float>(kpe_scale);
params.ckv_scale_arr =
maybe_ckv_scale_arr.has_value()
? static_cast<const float*>(maybe_ckv_scale_arr.value().data_ptr())
: nullptr;
params.return_lse_base_on_e = return_lse_base_on_e;

cudaError_t status = mla::BatchMLAPagedAttention<MASK_MODE, HEAD_DIM_CKV, HEAD_DIM_KPE>(
Expand Down
4 changes: 2 additions & 2 deletions csrc/batch_mla_sm90_binding.cu
Original file line number Diff line number Diff line change
Expand Up @@ -33,8 +33,8 @@ void BatchMLAPagedAttentionSM90Run(TensorView float_workspace_buffer,
TensorView kpe_cache, TensorView kv_indices, TensorView o,
Optional<TensorView> maybe_lse, int64_t mask_mode_code,
int64_t num_heads, int64_t page_size, double sm_scale,
bool return_lse_base_on_e, double ckv_scale,
double kpe_scale ADDITIONAL_FUNC_PARAMS);
bool return_lse_base_on_e, double ckv_scale, double kpe_scale,
Optional<TensorView> maybe_ckv_scale_arr ADDITIONAL_FUNC_PARAMS);

TVM_FFI_DLL_EXPORT_TYPED_FUNC(plan, BatchMLAPagedAttentionSM90Plan);
TVM_FFI_DLL_EXPORT_TYPED_FUNC(run, BatchMLAPagedAttentionSM90Run);
19 changes: 11 additions & 8 deletions csrc/batch_mla_sm90_run.cu
Original file line number Diff line number Diff line change
Expand Up @@ -26,14 +26,13 @@ using namespace flashinfer;
using tvm::ffi::Array;
using tvm::ffi::Optional;

void BatchMLAPagedAttentionSM90Run(TensorView float_workspace_buffer,
TensorView int_workspace_buffer, Array<int64_t> plan_info_vec,
TensorView q_nope, TensorView q_pe, TensorView ckv_cache,
TensorView kpe_cache, TensorView kv_indices, TensorView o,
Optional<TensorView> maybe_lse, int64_t mask_mode_code,
int64_t num_heads, int64_t page_size, double sm_scale,
bool return_lse_base_on_e, double ckv_scale,
double kpe_scale ADDITIONAL_FUNC_PARAMS) {
void BatchMLAPagedAttentionSM90Run(
TensorView float_workspace_buffer, TensorView int_workspace_buffer,
Array<int64_t> plan_info_vec, TensorView q_nope, TensorView q_pe, TensorView ckv_cache,
TensorView kpe_cache, TensorView kv_indices, TensorView o, Optional<TensorView> maybe_lse,
int64_t mask_mode_code, int64_t num_heads, int64_t page_size, double sm_scale,
bool return_lse_base_on_e, double ckv_scale, double kpe_scale,
Optional<TensorView> maybe_ckv_scale_arr ADDITIONAL_FUNC_PARAMS) {
// q_nope: [n, num_heads, head_dim_ckv]
// q_pe: [n, num_heads, head_dim_kpe]
// ckv_cache: [num_pages, page_size, head_dim_ckv]
Expand Down Expand Up @@ -119,6 +118,10 @@ void BatchMLAPagedAttentionSM90Run(TensorView float_workspace_buffer,
params.sm_scale = sm_scale;
params.ckv_scale = static_cast<float>(ckv_scale);
params.kpe_scale = static_cast<float>(kpe_scale);
params.ckv_scale_arr =
maybe_ckv_scale_arr.has_value()
? static_cast<const float*>(maybe_ckv_scale_arr.value().data_ptr())
: nullptr;

cudaError_t status =
mla::BatchMLAPageAttentionHopper<MASK_MODE, HEAD_DIM_CKV, HEAD_DIM_KPE>(
Expand Down
64 changes: 47 additions & 17 deletions flashinfer/mla/_core.py
Original file line number Diff line number Diff line change
Expand Up @@ -2165,7 +2165,7 @@ def plan(
head_dim_ckv : int
The head dimension of compressed-kv.
head_dim_kpe : int
The head dimension for rope k-cache.
The head dimension for rope k-cache. Zero is supported for NoPE MLA.
page_size : int
The page size of the paged kv-cache.
causal : bool
Expand All @@ -2188,32 +2188,30 @@ def plan(
f"MLA kv_data_type {kv_data_type} is not supported. "
f"Supported dtypes: {list(_SUPPORTED_MLA_KV_DTYPES)}."
)
if head_dim_kpe < 0:
raise ValueError(f"head_dim_kpe must be >= 0, got {head_dim_kpe}.")

Comment thread
coderabbitai[bot] marked this conversation as resolved.
Resolved
if kv_data_type == torch.float8_e4m3fn:
if self._backend != "fa3":
if self._backend not in ("fa2", "fa3"):
raise ValueError(
"FP8 kv_data_type for MLA is only supported with the fa3 "
"FP8 kv_data_type for MLA is only supported with the fa2 or fa3 "
f"backend on SM90, got backend={self._backend!r}."
)
# Backend selection is independent of the runtime device; FP8 MLA
# requires SM90 specifically.
major, minor = get_compute_capability(self.device)
if major != 9:
raise ValueError(
"FP8 kv_data_type for MLA requires an SM90 (Hopper) device, "
f"got SM{major}{minor}."
)
# Removing this guard exposes vec_cast<half, fp8_e4m3>, which
# exists but is untested for MLA — silent wrong output.
if q_data_type != torch.bfloat16:
raise ValueError(
"FP8 kv_data_type for MLA currently only supports "
f"q_data_type=torch.bfloat16, got {q_data_type}."
)
# Also enforced by static_assert in mla_hopper.cuh.
if head_dim_ckv != 512 or head_dim_kpe != 64:
if head_dim_ckv != 512 or head_dim_kpe not in (0, 64):
raise ValueError(
"FP8 kv_data_type for MLA currently only supports "
"head_dim_ckv=512 and head_dim_kpe=64 (DeepSeek MLA), got "
"head_dim_ckv=512 and head_dim_kpe in (0, 64), got "
f"head_dim_ckv={head_dim_ckv}, head_dim_kpe={head_dim_kpe}."
)

Expand Down Expand Up @@ -2244,6 +2242,7 @@ def plan(
self._causal = causal
self._page_size = page_size
self._sm_scale = sm_scale
self._head_dim_ckv = head_dim_ckv
# Used by run() to reject dtype mismatches; the C++ launcher
# reinterprets storage by the JIT-template type chosen at plan(),
# so a mismatch produces silent wrong output.
Expand Down Expand Up @@ -2281,6 +2280,7 @@ def run(
*,
ckv_scale: Optional[float] = None,
kpe_scale: Optional[float] = None,
ckv_scale_arr: Optional[torch.Tensor] = None,
) -> torch.Tensor: ...

@overload
Expand All @@ -2301,6 +2301,7 @@ def run(
*,
ckv_scale: Optional[float] = None,
kpe_scale: Optional[float] = None,
ckv_scale_arr: Optional[torch.Tensor] = None,
) -> Tuple[torch.Tensor, torch.Tensor]: ...

@flashinfer_api(trace=mla_paged_decode_trace)
Expand All @@ -2321,6 +2322,7 @@ def run(
*,
ckv_scale: Optional[float] = None,
kpe_scale: Optional[float] = None,
ckv_scale_arr: Optional[torch.Tensor] = None,

Copy link
Copy Markdown
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Non-blocking but just want to hear thoughts: the intention is that we will accept only one of either ckv_scale or ckv_scale_arr, correct?

  • Since these arguments are already keyword only, we are free to reorder them (i.e. putting ckv_scale_arr directly after ckv_scale)
  • would it be cleaner to only keep one argument as ckv_scale: Union[float, torch.Tensor]?

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yes, they are intended to be mutually exclusive. I reordered ckv_scale_arr directly after ckv_scale and now require exactly one of them for FP8 KV.

I kept them as separate arguments because the trace schema represents scalar and tensor inputs with different descriptors. A single Union[float, torch.Tensor] parameter cannot faithfully describe both forms in the generated benchmark definition. Internally, the grouped-scale path still passes 1.0 through the unused scalar launcher slot, so the C++ ABI remains unchanged.

) -> Union[torch.Tensor, Tuple[torch.Tensor, torch.Tensor]]:
r"""Run the MLA attention computation.

Expand All @@ -2335,7 +2337,7 @@ def run(
``head_dim_ckv`` is 512 in DeepSeek v2/v3 models.
kpe_cache : torch.Tensor
The rope part of the kv-cache tensor, shape: ``[num_pages, page_size, head_dim_kpe]``.
``head_dim_kpe`` is 64 in DeepSeek v2/v3 models.
``head_dim_kpe`` can be zero for NoPE MLA.
out : Optional[torch.Tensor]
The output tensor, if not provided, will be allocated internally.
When ``o_scale`` is provided, this should be an FP8 tensor.
Expand Down Expand Up @@ -2363,12 +2365,17 @@ def run(
Per-tensor dequantization scale for the compressed-KV cache when
``kv_data_type`` is FP8 (``real = quantized * ckv_scale``). Required
(together with ``kpe_scale``) for the FP8 KV cache path on the
``fa3`` backend. Must be a finite positive value. Must not be
``fa2`` or ``fa3`` backend. Must be a finite positive value. Must not be
provided when ``kv_data_type`` is BF16/FP16.
kpe_scale : Optional[float]
Per-tensor dequantization scale for the rope-K cache when
``kv_data_type`` is FP8 (``real = quantized * kpe_scale``). Same
usage rules as ``ckv_scale``.
ckv_scale_arr : Optional[torch.Tensor]
Per-token, per-128-channel CKV dequantization scales. The expected
shape is ``ckv_cache.shape[:-1] + (head_dim_ckv // 128,)`` and the
dtype must be contiguous float32. When provided, these values
override ``ckv_scale`` for CKV dequantization.
"""
if self._backend == "cutlass":
if return_lse:
Expand All @@ -2377,10 +2384,14 @@ def run(
raise ValueError(
"profiler_buffer does not support cutlass backend for now."
)
if ckv_scale is not None or kpe_scale is not None:
if (
ckv_scale is not None
or kpe_scale is not None
or ckv_scale_arr is not None
):
raise ValueError(
"ckv_scale / kpe_scale are only supported with the fa3 backend "
"and FP8 kv_data_type."
"ckv_scale / kpe_scale / ckv_scale_arr are only supported with "
"an fa2/fa3 backend and FP8 kv_data_type."
)
self._cached_module = get_mla_module()
output_scale = 1.0
Expand Down Expand Up @@ -2504,7 +2515,25 @@ def run(
lse, q_nope.shape[:2], torch.float32, q_nope.device, "lse"
)
profiler_args = (profiler_buffer,) if self._use_profiler else ()
self._cached_module.run(
if ckv_scale_arr is not None:
if not kv_is_fp8:
raise ValueError(
"ckv_scale_arr is only valid when kv_data_type is FP8."
)
expected_scale_shape = (
*ckv_cache.shape[:-1],
self._head_dim_ckv // 128,
)
check_shape_dtype_device(
ckv_scale_arr,
expected_scale_shape,
torch.float32,
ckv_cache.device,
"ckv_scale_arr",
)
if not ckv_scale_arr.is_contiguous():
raise ValueError("ckv_scale_arr must be contiguous.")
Comment thread
coderabbitai[bot] marked this conversation as resolved.
Resolved
run_args = (
self._float_workspace_buffer,
self._int_workspace_buffer,
self._plan_info,
Expand All @@ -2522,8 +2551,9 @@ def run(
return_lse_base_on_e,
ckv_scale_f,
kpe_scale_f,
*profiler_args,
ckv_scale_arr,
)
self._cached_module.run(*run_args, *profiler_args)

return (out, lse) if return_lse else out

Expand Down
17 changes: 14 additions & 3 deletions flashinfer/trace/templates/attention.py
Original file line number Diff line number Diff line change
Expand Up @@ -1425,6 +1425,7 @@ def _mla_paged_decode_init(
num_pages_per_seq: int = 4,
len_indptr: int = 0,
num_kv_indices: int = 0,
ckv_scale_groups: int = 0,
device: str = "cuda",
seed: int = 0,
):
Expand All @@ -1433,7 +1434,7 @@ def _mla_paged_decode_init(
Sourced from ``tests/trace/example.py`` MLA section. Default
``num_qo_heads=16`` matches DeepSeek-V3 TP=8.
"""
del num_pages, len_indptr, num_kv_indices
del num_pages, len_indptr, num_kv_indices, ckv_scale_groups
torch.manual_seed(seed)
total_pages = batch_size * num_pages_per_seq
qo_indptr = torch.arange(batch_size + 1, dtype=torch.int32, device=device)
Expand Down Expand Up @@ -1504,6 +1505,7 @@ def _mla_paged_decode_init(
"head_dim_ckv": Const(abbrev="ckv"),
"head_dim_kpe": Const(abbrev="kpe"),
"page_size": Const(abbrev="ps"),
"ckv_scale_groups": Var(description="Number of 128-channel CKV scale groups."),
"num_pages": Var(
description="Total number of allocated pages in the KV cache."
),
Expand Down Expand Up @@ -1551,7 +1553,7 @@ def _mla_paged_decode_init(
description=(
"Per-tensor dequantization scale for the compressed-KV cache when "
"kv_data_type is FP8 (real = quantized * ckv_scale). Required "
"together with kpe_scale for the FP8 KV cache path on the fa3 "
"together with kpe_scale for the FP8 KV cache path on the fa2/fa3 "
"backend. Set during run(), not plan()."
),
),
Expand All @@ -1561,10 +1563,19 @@ def _mla_paged_decode_init(
description=(
"Per-tensor dequantization scale for the rope-K cache when "
"kv_data_type is FP8 (real = quantized * kpe_scale). Required "
"together with ckv_scale for the FP8 KV cache path on the fa3 "
"together with ckv_scale for the FP8 KV cache path on the fa2/fa3 "
"backend. Set during run(), not plan()."
),
),
"ckv_scale_arr": Tensor(
["num_pages", "page_size", "ckv_scale_groups"],
dtype="float32",
optional=True,
description=(
"Optional per-token, per-128-channel CKV dequantization scales "
"for the FP8 KV cache path."
),
),
"return_lse": Scalar(
"int32", optional=True, description="Bool: also return LSE."
),
Expand Down
Loading
Loading