From 81b3c5cb3aad8a6195c764bb4a722d33b1228306 Mon Sep 17 00:00:00 2001 From: Xinyuan Tong Date: Thu, 6 Aug 2026 07:07:35 +0000 Subject: [PATCH 1/5] feat: support zero KPE dimension in native MLA --- flashinfer/mla/_core.py | 3 + include/flashinfer/attention/mla.cuh | 13 ++-- include/flashinfer/attention/mla_hopper.cuh | 11 +++- tests/attention/test_deepseek_mla.py | 68 +++++++++++++++++++++ 4 files changed, 89 insertions(+), 6 deletions(-) diff --git a/flashinfer/mla/_core.py b/flashinfer/mla/_core.py index d8c401b9c7e..29e3e0a6a04 100644 --- a/flashinfer/mla/_core.py +++ b/flashinfer/mla/_core.py @@ -2188,6 +2188,9 @@ def plan( f"MLA kv_data_type {kv_data_type} is not supported. " f"Supported dtypes: {list(_SUPPORTED_MLA_KV_DTYPES)}." ) + if head_dim_kpe < 0: + raise ValueError(f"head_dim_kpe must be >= 0, got {head_dim_kpe}.") + if kv_data_type == torch.float8_e4m3fn: if self._backend != "fa3": raise ValueError( diff --git a/include/flashinfer/attention/mla.cuh b/include/flashinfer/attention/mla.cuh index 17ac73b81f1..af4e4c2a500 100644 --- a/include/flashinfer/attention/mla.cuh +++ b/include/flashinfer/attention/mla.cuh @@ -489,10 +489,15 @@ __device__ __forceinline__ void compute_mla_qk(typename KTraits::SharedStorage* smem_t ckv_smem(smem_storage->ckv_smem[stage_idx]); smem_t kpe_smem(smem_storage->kpe_p_smem[stage_idx]); const uint32_t lane_idx = threadIdx.x, warpgroup_idx = threadIdx.z, warp_idx_in_wg = threadIdx.y; - compute_qk_(q_smem_pe, kpe_smem, s_frag); - compute_qk_(q_smem_nope, ckv_smem, s_frag); + if constexpr (KTraits::NUM_MMA_D_KPE > 0) { + compute_qk_(q_smem_pe, kpe_smem, s_frag); + compute_qk_(q_smem_nope, ckv_smem, s_frag); + } else { + compute_qk_(q_smem_nope, ckv_smem, s_frag); + } } template diff --git a/include/flashinfer/attention/mla_hopper.cuh b/include/flashinfer/attention/mla_hopper.cuh index 8093b2f937b..ac8cf6bbaf6 100644 --- a/include/flashinfer/attention/mla_hopper.cuh +++ b/include/flashinfer/attention/mla_hopper.cuh @@ -153,7 +153,8 @@ struct HopperKernelTraits static constexpr uint32_t CKV_B128_PER_ROW = HEAD_DIM_CKV_ * sizeof(DTypeKV_) / 16; static constexpr uint32_t KPE_B128_PER_ROW = HEAD_DIM_KPE_ * sizeof(DTypeKV_) / 16; static constexpr uint32_t LANES_PER_ROW_CKV = (CKV_B128_PER_ROW >= 8) ? 8 : CKV_B128_PER_ROW; - static constexpr uint32_t LANES_PER_ROW_KPE = (KPE_B128_PER_ROW >= 8) ? 8 : KPE_B128_PER_ROW; + static constexpr uint32_t LANES_PER_ROW_KPE = + KPE_B128_PER_ROW == 0 ? 1 : ((KPE_B128_PER_ROW >= 8) ? 8 : KPE_B128_PER_ROW); static constexpr uint32_t INNER_LOADS_CKV = CKV_B128_PER_ROW / LANES_PER_ROW_CKV; static constexpr uint32_t INNER_LOADS_KPE = KPE_B128_PER_ROW / LANES_PER_ROW_KPE; // Swizzle for the FP8 *raw* KPE shmem storage. When KPE_B128_PER_ROW < 8 (FP8 @@ -465,8 +466,14 @@ __device__ __forceinline__ void compute_mla_qk(typename KTraits::SharedStorage* make_smem_desc(ckv_smem_ptr); + if constexpr (KTraits::NUM_MMA_D_KPE == 0) { + wgmma::op(desc_q_nope, desc_ckv, s_frag); + desc_q_nope += 2; + desc_ckv += 2; + } #pragma unroll - for (uint32_t mma_d_ckv = 0; mma_d_ckv < KTraits::NUM_MMA_D_CKV; ++mma_d_ckv) { + for (uint32_t mma_d_ckv = KTraits::NUM_MMA_D_KPE == 0 ? 1 : 0; mma_d_ckv < KTraits::NUM_MMA_D_CKV; + ++mma_d_ckv) { wgmma::op(desc_q_nope, desc_ckv, s_frag); if ((mma_d_ckv + 1) % 4 == 0) { desc_q_nope += 64 - 6; diff --git a/tests/attention/test_deepseek_mla.py b/tests/attention/test_deepseek_mla.py index e291077742f..d7c14978e78 100644 --- a/tests/attention/test_deepseek_mla.py +++ b/tests/attention/test_deepseek_mla.py @@ -277,6 +277,74 @@ def generate_kv_from_cache(ckv, kpe, kv_len, batch_size, num_heads): return k, v +@pytest.mark.parametrize("backend", ["fa2", "fa3"]) +def test_batch_mla_without_kpe(backend): + device = torch.device("cuda:0") + if backend == "fa3" and not is_sm90a_supported(device): + pytest.skip("FA3 is not supported on this device") + + torch.manual_seed(42) + batch_size = 2 + qo_len = 3 + kv_len = 97 + num_heads = 16 + head_dim_ckv = 512 + head_dim_kpe = 0 + page_size = 16 + dtype = torch.float16 + pages_num = math.ceil(kv_len / page_size) + q_nope = torch.randn( + batch_size * qo_len, num_heads, head_dim_ckv, dtype=dtype, device=device + ) + q_pe = torch.empty( + batch_size * qo_len, num_heads, head_dim_kpe, dtype=dtype, device=device + ) + ckv = torch.randn( + batch_size * pages_num, + page_size, + head_dim_ckv, + dtype=dtype, + device=device, + ) + kpe = torch.empty( + batch_size * pages_num, + page_size, + head_dim_kpe, + dtype=dtype, + device=device, + ) + sm_scale = 1.0 / math.sqrt(128) + workspace = torch.empty(128 * 1024 * 1024, dtype=torch.int8, device=device) + wrapper = flashinfer.mla.BatchMLAPagedAttentionWrapper(workspace, backend=backend) + q_indptr = torch.arange(batch_size + 1, device=device, dtype=torch.int32) * qo_len + kv_indptr = ( + torch.arange(batch_size + 1, device=device, dtype=torch.int32) * pages_num + ) + kv_indices = torch.arange(batch_size * pages_num, device=device, dtype=torch.int32) + kv_lens = torch.full((batch_size,), kv_len, dtype=torch.int32, device=device) + + wrapper.plan( + q_indptr, + kv_indptr, + kv_indices, + kv_lens, + num_heads, + head_dim_ckv, + head_dim_kpe, + page_size, + False, + sm_scale, + dtype, + dtype, + ) + output, lse = wrapper.run(q_nope, q_pe, ckv, kpe, return_lse=True) + + key, value = generate_kv_from_cache(ckv, kpe, kv_len, batch_size, num_heads) + output_ref, lse_ref = attention_ref(batch_size, q_nope, key, value, False, sm_scale) + torch.testing.assert_close(output, output_ref, rtol=1e-3, atol=1e-3) + torch.testing.assert_close(lse, lse_ref.flatten(0, 1), rtol=1e-3, atol=1e-3) + + @pytest.mark.parametrize("batch_size", [1, 3, 5, 7]) @pytest.mark.parametrize("kv_len_0", [0, 1, 3, 11]) @pytest.mark.parametrize("kv_len_1", [17, 33, 79, 114]) From bebda8090801b474a11187ac994eb5506f7903bd Mon Sep 17 00:00:00 2001 From: Xinyuan Tong Date: Wed, 12 Aug 2026 15:36:16 +0000 Subject: [PATCH 2/5] feat: support FP8 KV NoPE MLA on SM90 --- csrc/batch_mla_binding.cu | 3 +- csrc/batch_mla_run.cu | 7 +- csrc/batch_mla_sm90_binding.cu | 4 +- csrc/batch_mla_sm90_run.cu | 19 ++- flashinfer/mla/_core.py | 58 +++++-- include/flashinfer/attention/mla.cuh | 180 +++++++++++++++++--- include/flashinfer/attention/mla_hopper.cuh | 100 +++++++---- include/flashinfer/attention/mla_params.cuh | 2 + tests/attention/test_deepseek_mla.py | 111 +++++++++++- 9 files changed, 399 insertions(+), 85 deletions(-) diff --git a/csrc/batch_mla_binding.cu b/csrc/batch_mla_binding.cu index 93581a528db..291854c8bc4 100644 --- a/csrc/batch_mla_binding.cu +++ b/csrc/batch_mla_binding.cu @@ -32,7 +32,8 @@ void BatchMLAPagedAttentionRun(TensorView float_workspace_buffer, TensorView int TensorView ckv_cache, TensorView kpe_cache, TensorView kv_indices, TensorView o, Optional maybe_lse, int64_t mask_mode_code, int64_t num_heads, int64_t page_size, double sm_scale, - bool return_lse_base_on_e, double ckv_scale, double kpe_scale); + bool return_lse_base_on_e, double ckv_scale, double kpe_scale, + Optional maybe_ckv_scale_arr); TVM_FFI_DLL_EXPORT_TYPED_FUNC(plan, BatchMLAPagedAttentionPlan); TVM_FFI_DLL_EXPORT_TYPED_FUNC(run, BatchMLAPagedAttentionRun); diff --git a/csrc/batch_mla_run.cu b/csrc/batch_mla_run.cu index 6d28b68cec4..e24d17f2975 100644 --- a/csrc/batch_mla_run.cu +++ b/csrc/batch_mla_run.cu @@ -32,7 +32,8 @@ void BatchMLAPagedAttentionRun(TensorView float_workspace_buffer, TensorView int TensorView ckv_cache, TensorView kpe_cache, TensorView kv_indices, TensorView o, Optional maybe_lse, int64_t mask_mode_code, int64_t num_heads, int64_t page_size, double sm_scale, - bool return_lse_base_on_e, double ckv_scale, double kpe_scale) { + bool return_lse_base_on_e, double ckv_scale, double kpe_scale, + Optional maybe_ckv_scale_arr) { // q_nope: [n, num_heads, head_dim_ckv] // q_pe: [n, num_heads, head_dim_kpe] // ckv_cache: [num_pages, page_size, head_dim_ckv] @@ -117,6 +118,10 @@ void BatchMLAPagedAttentionRun(TensorView float_workspace_buffer, TensorView int params.sm_scale = sm_scale; params.ckv_scale = static_cast(ckv_scale); params.kpe_scale = static_cast(kpe_scale); + params.ckv_scale_arr = + maybe_ckv_scale_arr.has_value() + ? static_cast(maybe_ckv_scale_arr.value().data_ptr()) + : nullptr; params.return_lse_base_on_e = return_lse_base_on_e; cudaError_t status = mla::BatchMLAPagedAttention( diff --git a/csrc/batch_mla_sm90_binding.cu b/csrc/batch_mla_sm90_binding.cu index b1a8ff650b2..91b586658f2 100644 --- a/csrc/batch_mla_sm90_binding.cu +++ b/csrc/batch_mla_sm90_binding.cu @@ -33,8 +33,8 @@ void BatchMLAPagedAttentionSM90Run(TensorView float_workspace_buffer, TensorView kpe_cache, TensorView kv_indices, TensorView o, Optional maybe_lse, int64_t mask_mode_code, int64_t num_heads, int64_t page_size, double sm_scale, - bool return_lse_base_on_e, double ckv_scale, - double kpe_scale ADDITIONAL_FUNC_PARAMS); + bool return_lse_base_on_e, double ckv_scale, double kpe_scale, + Optional maybe_ckv_scale_arr ADDITIONAL_FUNC_PARAMS); TVM_FFI_DLL_EXPORT_TYPED_FUNC(plan, BatchMLAPagedAttentionSM90Plan); TVM_FFI_DLL_EXPORT_TYPED_FUNC(run, BatchMLAPagedAttentionSM90Run); diff --git a/csrc/batch_mla_sm90_run.cu b/csrc/batch_mla_sm90_run.cu index f7f26410475..7a5cd4f2ec8 100644 --- a/csrc/batch_mla_sm90_run.cu +++ b/csrc/batch_mla_sm90_run.cu @@ -26,14 +26,13 @@ using namespace flashinfer; using tvm::ffi::Array; using tvm::ffi::Optional; -void BatchMLAPagedAttentionSM90Run(TensorView float_workspace_buffer, - TensorView int_workspace_buffer, Array plan_info_vec, - TensorView q_nope, TensorView q_pe, TensorView ckv_cache, - TensorView kpe_cache, TensorView kv_indices, TensorView o, - Optional maybe_lse, int64_t mask_mode_code, - int64_t num_heads, int64_t page_size, double sm_scale, - bool return_lse_base_on_e, double ckv_scale, - double kpe_scale ADDITIONAL_FUNC_PARAMS) { +void BatchMLAPagedAttentionSM90Run( + TensorView float_workspace_buffer, TensorView int_workspace_buffer, + Array plan_info_vec, TensorView q_nope, TensorView q_pe, TensorView ckv_cache, + TensorView kpe_cache, TensorView kv_indices, TensorView o, Optional maybe_lse, + int64_t mask_mode_code, int64_t num_heads, int64_t page_size, double sm_scale, + bool return_lse_base_on_e, double ckv_scale, double kpe_scale, + Optional maybe_ckv_scale_arr ADDITIONAL_FUNC_PARAMS) { // q_nope: [n, num_heads, head_dim_ckv] // q_pe: [n, num_heads, head_dim_kpe] // ckv_cache: [num_pages, page_size, head_dim_ckv] @@ -119,6 +118,10 @@ void BatchMLAPagedAttentionSM90Run(TensorView float_workspace_buffer, params.sm_scale = sm_scale; params.ckv_scale = static_cast(ckv_scale); params.kpe_scale = static_cast(kpe_scale); + params.ckv_scale_arr = + maybe_ckv_scale_arr.has_value() + ? static_cast(maybe_ckv_scale_arr.value().data_ptr()) + : nullptr; cudaError_t status = mla::BatchMLAPageAttentionHopper( diff --git a/flashinfer/mla/_core.py b/flashinfer/mla/_core.py index f4684996f8c..46f8073b0ae 100644 --- a/flashinfer/mla/_core.py +++ b/flashinfer/mla/_core.py @@ -2165,7 +2165,7 @@ def plan( head_dim_ckv : int The head dimension of compressed-kv. head_dim_kpe : int - The head dimension for rope k-cache. + The head dimension for rope k-cache. Zero is supported for NoPE MLA. page_size : int The page size of the paged kv-cache. causal : bool @@ -2192,31 +2192,26 @@ def plan( raise ValueError(f"head_dim_kpe must be >= 0, got {head_dim_kpe}.") if kv_data_type == torch.float8_e4m3fn: - if self._backend != "fa3": + if self._backend not in ("fa2", "fa3"): raise ValueError( - "FP8 kv_data_type for MLA is only supported with the fa3 " + "FP8 kv_data_type for MLA is only supported with the fa2 or fa3 " f"backend on SM90, got backend={self._backend!r}." ) - # Backend selection is independent of the runtime device; FP8 MLA - # requires SM90 specifically. major, minor = get_compute_capability(self.device) if major != 9: raise ValueError( "FP8 kv_data_type for MLA requires an SM90 (Hopper) device, " f"got SM{major}{minor}." ) - # Removing this guard exposes vec_cast, which - # exists but is untested for MLA — silent wrong output. if q_data_type != torch.bfloat16: raise ValueError( "FP8 kv_data_type for MLA currently only supports " f"q_data_type=torch.bfloat16, got {q_data_type}." ) - # Also enforced by static_assert in mla_hopper.cuh. - if head_dim_ckv != 512 or head_dim_kpe != 64: + if head_dim_ckv != 512 or head_dim_kpe not in (0, 64): raise ValueError( "FP8 kv_data_type for MLA currently only supports " - "head_dim_ckv=512 and head_dim_kpe=64 (DeepSeek MLA), got " + "head_dim_ckv=512 and head_dim_kpe in (0, 64), got " f"head_dim_ckv={head_dim_ckv}, head_dim_kpe={head_dim_kpe}." ) @@ -2247,6 +2242,7 @@ def plan( self._causal = causal self._page_size = page_size self._sm_scale = sm_scale + self._head_dim_ckv = head_dim_ckv # Used by run() to reject dtype mismatches; the C++ launcher # reinterprets storage by the JIT-template type chosen at plan(), # so a mismatch produces silent wrong output. @@ -2284,6 +2280,7 @@ def run( *, ckv_scale: Optional[float] = None, kpe_scale: Optional[float] = None, + ckv_scale_arr: Optional[torch.Tensor] = None, ) -> torch.Tensor: ... @overload @@ -2304,6 +2301,7 @@ def run( *, ckv_scale: Optional[float] = None, kpe_scale: Optional[float] = None, + ckv_scale_arr: Optional[torch.Tensor] = None, ) -> Tuple[torch.Tensor, torch.Tensor]: ... @flashinfer_api(trace=mla_paged_decode_trace) @@ -2324,6 +2322,7 @@ def run( *, ckv_scale: Optional[float] = None, kpe_scale: Optional[float] = None, + ckv_scale_arr: Optional[torch.Tensor] = None, ) -> Union[torch.Tensor, Tuple[torch.Tensor, torch.Tensor]]: r"""Run the MLA attention computation. @@ -2338,7 +2337,7 @@ def run( ``head_dim_ckv`` is 512 in DeepSeek v2/v3 models. kpe_cache : torch.Tensor The rope part of the kv-cache tensor, shape: ``[num_pages, page_size, head_dim_kpe]``. - ``head_dim_kpe`` is 64 in DeepSeek v2/v3 models. + ``head_dim_kpe`` can be zero for NoPE MLA. out : Optional[torch.Tensor] The output tensor, if not provided, will be allocated internally. When ``o_scale`` is provided, this should be an FP8 tensor. @@ -2366,12 +2365,17 @@ def run( Per-tensor dequantization scale for the compressed-KV cache when ``kv_data_type`` is FP8 (``real = quantized * ckv_scale``). Required (together with ``kpe_scale``) for the FP8 KV cache path on the - ``fa3`` backend. Must be a finite positive value. Must not be + ``fa2`` or ``fa3`` backend. Must be a finite positive value. Must not be provided when ``kv_data_type`` is BF16/FP16. kpe_scale : Optional[float] Per-tensor dequantization scale for the rope-K cache when ``kv_data_type`` is FP8 (``real = quantized * kpe_scale``). Same usage rules as ``ckv_scale``. + ckv_scale_arr : Optional[torch.Tensor] + Per-token, per-128-channel CKV dequantization scales. The expected + shape is ``ckv_cache.shape[:-1] + (head_dim_ckv // 128,)`` and the + dtype must be contiguous float32. When provided, these values + override ``ckv_scale`` for CKV dequantization. """ if self._backend == "cutlass": if return_lse: @@ -2380,10 +2384,14 @@ def run( raise ValueError( "profiler_buffer does not support cutlass backend for now." ) - if ckv_scale is not None or kpe_scale is not None: + if ( + ckv_scale is not None + or kpe_scale is not None + or ckv_scale_arr is not None + ): raise ValueError( - "ckv_scale / kpe_scale are only supported with the fa3 backend " - "and FP8 kv_data_type." + "ckv_scale / kpe_scale / ckv_scale_arr are only supported with " + "an fa2/fa3 backend and FP8 kv_data_type." ) self._cached_module = get_mla_module() output_scale = 1.0 @@ -2507,7 +2515,22 @@ def run( lse, q_nope.shape[:2], torch.float32, q_nope.device, "lse" ) profiler_args = (profiler_buffer,) if self._use_profiler else () - self._cached_module.run( + if ckv_scale_arr is not None: + if not kv_is_fp8: + raise ValueError( + "ckv_scale_arr is only valid when kv_data_type is FP8." + ) + expected_scale_shape = ckv_cache.shape[:-1] + (self._head_dim_ckv // 128,) + check_shape_dtype_device( + ckv_scale_arr, + expected_scale_shape, + torch.float32, + ckv_cache.device, + "ckv_scale_arr", + ) + if not ckv_scale_arr.is_contiguous(): + raise ValueError("ckv_scale_arr must be contiguous.") + run_args = ( self._float_workspace_buffer, self._int_workspace_buffer, self._plan_info, @@ -2525,8 +2548,9 @@ def run( return_lse_base_on_e, ckv_scale_f, kpe_scale_f, - *profiler_args, + ckv_scale_arr, ) + self._cached_module.run(*run_args, *profiler_args) return (out, lse) if return_lse else out diff --git a/include/flashinfer/attention/mla.cuh b/include/flashinfer/attention/mla.cuh index af4e4c2a500..876dc3ce743 100644 --- a/include/flashinfer/attention/mla.cuh +++ b/include/flashinfer/attention/mla.cuh @@ -32,10 +32,9 @@ namespace mla { struct StandardAttention : AttentionVariantBase { float sm_scale_log2; - // Per-tensor symmetric-quantization scales for the FP8 KV path. Both 1.0 on - // the BF16/FP16 path so they are no-ops there. float ckv_scale; float kpe_scale; + const float* ckv_scale_arr; PROFILER_CLOSURE_PARAMS_DECL @@ -45,12 +44,14 @@ struct StandardAttention : AttentionVariantBase { sm_scale_log2 = params.sm_scale * math::log2e; ckv_scale = params.ckv_scale; kpe_scale = params.kpe_scale; + ckv_scale_arr = params.ckv_scale_arr; } }; template struct SharedStorageQKVO { + static constexpr bool USE_KV_REPACK = std::is_same_v; union { struct { alignas(16) DTypeQ q_smem_nope[CTA_TILE_Q * HEAD_DIM_CKV]; @@ -59,6 +60,11 @@ struct SharedStorageQKVO { alignas(16) DTypeKV kpe_p_smem[NUM_STAGES] [CTA_TILE_KV * (HEAD_DIM_KPE > CTA_TILE_Q ? HEAD_DIM_KPE : CTA_TILE_Q)]; + alignas(16) std::conditional_t ckv_bf16; + alignas(16) std::conditional_t< + USE_KV_REPACK, DTypeQ[NUM_STAGES * CTA_TILE_KV * (HEAD_DIM_KPE > 0 ? HEAD_DIM_KPE : 1)], + DTypeQ[1]> kpe_bf16; union { alignas(16) float m_wg[2][CTA_TILE_Q]; // cross warpgroup synchronization alignas(16) float d_wg[2][CTA_TILE_Q]; // cross warpgroup synchronization @@ -102,6 +108,12 @@ struct KernelTraits { static constexpr uint32_t UPCAST_STRIDE_FINAL_O = HEAD_DIM_CKV / upcast_size(); static constexpr uint32_t UPCAST_STRIDE_P = CTA_TILE_KV / upcast_size(); + static constexpr bool USE_KV_REPACK = std::is_same_v; + static constexpr uint32_t UPCAST_STRIDE_CKV_BF16 = HEAD_DIM_CKV / upcast_size(); + static constexpr uint32_t UPCAST_STRIDE_KPE_BF16 = + (HEAD_DIM_KPE > 0 ? HEAD_DIM_KPE : 16) / upcast_size(); + static constexpr uint32_t UPCAST_STRIDE_P_BF16 = CTA_TILE_KV / upcast_size(); + using DTypeQ = DTypeQ_; using DTypeKV = DTypeKV_; using DTypeO = DTypeO_; @@ -221,7 +233,7 @@ __device__ __forceinline__ void load_kv( r * kpe_stride_n + (lane_idx % 8) * upcast_size(); #pragma unroll - for (uint32_t mma_d = 0; mma_d < KTraits::NUM_MMA_D_CKV / 4; ++mma_d) { + for (uint32_t mma_d = 0; mma_d < KTraits::UPCAST_STRIDE_CKV / 8; ++mma_d) { uint32_t ckv_smem_offset_w = ckv_smem.template get_permuted_offset( warp_idx_in_wg * 4 + lane_idx / 8, 8 * mma_d + lane_idx % 8); ckv_smem.load_128b_async(ckv_smem_offset_w, ckv_ptr, @@ -259,7 +271,7 @@ __device__ __forceinline__ void load_kv( r * kpe_stride_n + (lane_idx % 8) * upcast_size(); #pragma unroll - for (uint32_t mma_d = 0; mma_d < KTraits::NUM_MMA_D_CKV / 4; ++mma_d) { + for (uint32_t mma_d = 0; mma_d < KTraits::UPCAST_STRIDE_CKV / 8; ++mma_d) { uint32_t ckv_smem_offset_w = ckv_smem.template get_permuted_offset( 32 * mma_kv + warpgroup_idx * 16 + warp_idx_in_wg * 4 + lane_idx / 8, 8 * mma_d + lane_idx % 8); @@ -281,6 +293,86 @@ __device__ __forceinline__ void load_kv( } } +template +__device__ __forceinline__ void repack_fp8_kv_to_bf16( + typename KTraits::SharedStorage* smem_storage, const uint32_t stage_idx, float ckv_scale, + float kpe_scale, const float* ckv_scale_arr, uint32_t row_base, + const typename KTraits::IdType* kv_indices, uint_fastdiv block_size, uint32_t packed_kv_bound) { + using DTypeKV = typename KTraits::DTypeKV; + using DTypeQ = typename KTraits::DTypeQ; + constexpr uint32_t CTA_TILE_KV = KTraits::CTA_TILE_KV; + constexpr uint32_t HEAD_DIM_CKV = KTraits::HEAD_DIM_CKV; + constexpr uint32_t HEAD_DIM_KPE = KTraits::HEAD_DIM_KPE; + constexpr uint32_t FP8_COLS_CKV = HEAD_DIM_CKV / upcast_size(); + constexpr uint32_t NUM_B128_CKV = CTA_TILE_KV * FP8_COLS_CKV; + const uint32_t tid = threadIdx.x + threadIdx.y * 32 + threadIdx.z * 128; + constexpr uint32_t num_threads = KTraits::NUM_THREADS; + + using packed2_t = nv_bfloat162; + smem_t ckv_src(smem_storage->ckv_smem[stage_idx]); + smem_t ckv_dst(smem_storage->ckv_bf16 + + stage_idx * CTA_TILE_KV * HEAD_DIM_CKV); +#pragma unroll + for (uint32_t idx = tid; idx < NUM_B128_CKV; idx += num_threads) { + uint32_t row = idx / FP8_COLS_CKV, col = idx % FP8_COLS_CKV; + b128_t packed = + ckv_src.base[ckv_src.template get_permuted_offset(row, col)]; + alignas(16) DTypeQ conv[16]; + vec_cast::template cast<16>(conv, (DTypeKV*)&packed); + float scale = ckv_scale; + if (ckv_scale_arr != nullptr) { + uint32_t page, offset; + block_size.divmod(row_base + row, page, offset); + uint32_t physical_row = + (row_base + row) < packed_kv_bound ? kv_indices[page] * block_size + offset : 0; + scale = + ckv_scale_arr[physical_row * (HEAD_DIM_CKV / 128) + col / (128 / upcast_size())]; + } + if (ckv_scale_arr != nullptr) { +#pragma unroll + for (uint32_t k = 0; k < 16; ++k) { + conv[k] = static_cast(static_cast(conv[k]) * scale); + } + } else { + packed2_t scale_packed{static_cast(scale), static_cast(scale)}; +#pragma unroll + for (uint32_t k = 0; k < 8; ++k) { + ((packed2_t*)&conv[0])[k] = __hmul2(((packed2_t*)&conv[0])[k], scale_packed); + } + } + ckv_dst + .base[ckv_dst.template get_permuted_offset(row, 2 * col)] = + *(b128_t*)&conv[0]; + ckv_dst.base[ckv_dst.template get_permuted_offset( + row, 2 * col + 1)] = *(b128_t*)&conv[8]; + } + + if constexpr (HEAD_DIM_KPE > 0) { + constexpr uint32_t FP8_COLS_KPE = HEAD_DIM_KPE / upcast_size(); + constexpr uint32_t NUM_B128_KPE = CTA_TILE_KV * FP8_COLS_KPE; + packed2_t kpe_scale_packed{static_cast(kpe_scale), static_cast(kpe_scale)}; + smem_t kpe_src(smem_storage->kpe_p_smem[stage_idx]); + smem_t kpe_dst(smem_storage->kpe_bf16 + + stage_idx * CTA_TILE_KV * HEAD_DIM_KPE); +#pragma unroll + for (uint32_t idx = tid; idx < NUM_B128_KPE; idx += num_threads) { + uint32_t row = idx / FP8_COLS_KPE, col = idx % FP8_COLS_KPE; + b128_t packed = + kpe_src.base[kpe_src.template get_permuted_offset(row, col)]; + alignas(16) DTypeQ conv[16]; + vec_cast::template cast<16>(conv, (DTypeKV*)&packed); +#pragma unroll + for (uint32_t k = 0; k < 8; ++k) { + ((packed2_t*)&conv[0])[k] = __hmul2(((packed2_t*)&conv[0])[k], kpe_scale_packed); + } + kpe_dst.base[kpe_dst.template get_permuted_offset( + row, 2 * col)] = *(b128_t*)&conv[0]; + kpe_dst.base[kpe_dst.template get_permuted_offset( + row, 2 * col + 1)] = *(b128_t*)&conv[8]; + } + } +} + template __device__ __forceinline__ void compute_qk_(smem_t q_smem, @@ -481,22 +573,36 @@ __device__ __forceinline__ void compute_mla_qk(typename KTraits::SharedStorage* typename KTraits::DTypeQKAccum (*s_frag)[8]) { constexpr uint32_t UPCAST_STRIDE_Q_NOPE = KTraits::UPCAST_STRIDE_Q_NOPE; constexpr uint32_t UPCAST_STRIDE_Q_PE = KTraits::UPCAST_STRIDE_Q_PE; - constexpr uint32_t UPCAST_STRIDE_CKV = KTraits::UPCAST_STRIDE_CKV; - constexpr uint32_t UPCAST_STRIDE_KPE = KTraits::UPCAST_STRIDE_KPE; constexpr uint32_t NUM_MMA_KV = KTraits::NUM_MMA_KV; smem_t q_smem_nope(smem_storage->q_smem_nope); smem_t q_smem_pe(smem_storage->q_smem_pe); - smem_t ckv_smem(smem_storage->ckv_smem[stage_idx]); - smem_t kpe_smem(smem_storage->kpe_p_smem[stage_idx]); const uint32_t lane_idx = threadIdx.x, warpgroup_idx = threadIdx.z, warp_idx_in_wg = threadIdx.y; - if constexpr (KTraits::NUM_MMA_D_KPE > 0) { - compute_qk_(q_smem_pe, kpe_smem, s_frag); - compute_qk_(q_smem_nope, ckv_smem, s_frag); + if constexpr (KTraits::USE_KV_REPACK) { + smem_t ckv_smem( + smem_storage->ckv_bf16 + stage_idx * KTraits::CTA_TILE_KV * KTraits::HEAD_DIM_CKV); + if constexpr (KTraits::NUM_MMA_D_KPE > 0) { + smem_t kpe_smem( + smem_storage->kpe_bf16 + stage_idx * KTraits::CTA_TILE_KV * KTraits::HEAD_DIM_KPE); + compute_qk_(q_smem_pe, kpe_smem, s_frag); + compute_qk_(q_smem_nope, ckv_smem, s_frag); + } else { + compute_qk_(q_smem_nope, ckv_smem, s_frag); + } } else { - compute_qk_(q_smem_nope, ckv_smem, s_frag); + smem_t ckv_smem(smem_storage->ckv_smem[stage_idx]); + smem_t kpe_smem(smem_storage->kpe_p_smem[stage_idx]); + if constexpr (KTraits::NUM_MMA_D_KPE > 0) { + compute_qk_(q_smem_pe, kpe_smem, s_frag); + compute_qk_(q_smem_nope, ckv_smem, s_frag); + } else { + compute_qk_(q_smem_nope, ckv_smem, s_frag); + } } } @@ -509,8 +615,13 @@ __device__ __forceinline__ void compute_mla_pv(typename KTraits::SharedStorage* const uint32_t lane_idx = threadIdx.x, warpgroup_idx = threadIdx.z, warp_idx_in_wg = threadIdx.y; constexpr uint32_t NUM_MMA_KV = KTraits::NUM_MMA_KV; constexpr uint32_t NUM_MMA_D_CKV = KTraits::NUM_MMA_D_CKV; - constexpr uint32_t UPCAST_STRIDE_CKV = KTraits::UPCAST_STRIDE_CKV; - smem_t ckv_smem(smem_storage->ckv_smem[stage_idx]); + constexpr uint32_t UPCAST_STRIDE_CKV = + KTraits::USE_KV_REPACK ? KTraits::UPCAST_STRIDE_CKV_BF16 : KTraits::UPCAST_STRIDE_CKV; + smem_t ckv_smem( + KTraits::USE_KV_REPACK + ? reinterpret_cast(smem_storage->ckv_bf16 + + stage_idx * KTraits::CTA_TILE_KV * KTraits::HEAD_DIM_CKV) + : reinterpret_cast(smem_storage->ckv_smem[stage_idx])); uint32_t ckv_smem_offset_r = ckv_smem.template get_permuted_offset( lane_idx % 16, warpgroup_idx * NUM_MMA_D_CKV + lane_idx / 16); if constexpr (KTraits::QK_SHARD) { @@ -570,10 +681,10 @@ __device__ __forceinline__ void compute_mla_pv(typename KTraits::SharedStorage* } } else { // no need to store p_smem because all warpgroups are working on the same p - alignas(16) typename KTraits::DTypeKV p_f16[NUM_MMA_KV][8]; + alignas(16) typename KTraits::DTypeQ p_f16[NUM_MMA_KV][8]; #pragma unroll for (uint32_t mma_kv = 0; mma_kv < NUM_MMA_KV; ++mma_kv) { - vec_cast::cast<8>(p_f16[mma_kv], s_frag[mma_kv]); + vec_cast::cast<8>(p_f16[mma_kv], s_frag[mma_kv]); mma::m16k16_rowsum_f16f16f32(d, p_f16[mma_kv]); } #pragma unroll @@ -582,7 +693,7 @@ __device__ __forceinline__ void compute_mla_pv(typename KTraits::SharedStorage* for (uint32_t mma_d = 0; mma_d < NUM_MMA_D_CKV / 2; ++mma_d) { uint32_t v_frag[4]; ckv_smem.ldmatrix_m8n8x4_trans(ckv_smem_offset_r, v_frag); - mma::mma_sync_m16n16k16_row_col_f16f16f32( + mma::mma_sync_m16n16k16_row_col_f16f16f32( o_frag[mma_d], (uint32_t*)p_f16[mma_kv], v_frag); ckv_smem_offset_r = ckv_smem.template advance_offset_by_column<2>(ckv_smem_offset_r, mma_d); } @@ -983,6 +1094,13 @@ __global__ __launch_bounds__(KTraits::NUM_THREADS) void BatchMLAPagedAttentionKe __syncthreads(); // compute mla qk + if constexpr (KTraits::USE_KV_REPACK) { + repack_fp8_kv_to_bf16(&smem_storage, kv_tile_idx % NUM_STAGES, params.ckv_scale, + params.kpe_scale, params.ckv_scale_arr, + block_iter_base + kv_tile_idx * CTA_TILE_KV, kv_indices, + block_size, packed_kv_bound); + __syncthreads(); + } compute_mla_qk(&smem_storage, kv_tile_idx % NUM_STAGES, s_frag); // logits mask @@ -1013,6 +1131,13 @@ __global__ __launch_bounds__(KTraits::NUM_THREADS) void BatchMLAPagedAttentionKe __syncthreads(); // compute mla qk + if constexpr (KTraits::USE_KV_REPACK) { + repack_fp8_kv_to_bf16(&smem_storage, kv_tile_idx % NUM_STAGES, params.ckv_scale, + params.kpe_scale, params.ckv_scale_arr, + block_iter_base + kv_tile_idx * CTA_TILE_KV, kv_indices, + block_size, packed_kv_bound); + __syncthreads(); + } compute_mla_qk(&smem_storage, kv_tile_idx % NUM_STAGES, s_frag); // compute m,d states in online softmax @@ -1035,6 +1160,13 @@ __global__ __launch_bounds__(KTraits::NUM_THREADS) void BatchMLAPagedAttentionKe #pragma unroll for (; kv_tile_idx >= 0; --kv_tile_idx) { // compute mla qk + if constexpr (KTraits::USE_KV_REPACK) { + repack_fp8_kv_to_bf16(&smem_storage, kv_tile_idx % NUM_STAGES, params.ckv_scale, + params.kpe_scale, params.ckv_scale_arr, + block_iter_base + kv_tile_idx * CTA_TILE_KV, kv_indices, + block_size, packed_kv_bound); + __syncthreads(); + } compute_mla_qk(&smem_storage, kv_tile_idx % NUM_STAGES, s_frag); logits_mask_(qo_packed_idx_base, kv_start + kv_tile_idx * CTA_TILE_KV, q_len, kv_len, @@ -1119,8 +1251,12 @@ cudaError_t BatchMLAPagedAttention(Params params, uint32_t num_blks_x, uint32_t cudaDeviceGetAttribute(&smem_limit_per_sm, cudaDevAttrMaxSharedMemoryPerMultiprocessor, device); DISPATCH_SMEM_CONFIG(smem_limit_per_sm, NUM_STAGES, CTA_TILE_KV, QK_SHARD, { - using KTraits = KernelTraits; + constexpr uint32_t EFF_NUM_STAGES = NUM_STAGES; + constexpr uint32_t EFF_CTA_TILE_KV = std::is_same_v ? 32 : CTA_TILE_KV; + constexpr bool EFF_QK_SHARD = std::is_same_v ? false : QK_SHARD; + using KTraits = + KernelTraits; size_t smem_size = sizeof(typename KTraits::SharedStorage); auto kernel = BatchMLAPagedAttentionKernel; void* args[] = {(void*)¶ms}; diff --git a/include/flashinfer/attention/mla_hopper.cuh b/include/flashinfer/attention/mla_hopper.cuh index ac8cf6bbaf6..85a5f19b734 100644 --- a/include/flashinfer/attention/mla_hopper.cuh +++ b/include/flashinfer/attention/mla_hopper.cuh @@ -105,7 +105,10 @@ struct HopperSharedStorageQKVO { // path these collapse to a single element via std::conditional_t. alignas(16) std::conditional_t ckv_bf16; - alignas(16) std::conditional_t 0 ? CTA_TILE_KV* HEAD_DIM_KPE : 1], DTypeQ[1]> kpe_bf16; }; alignas(16) DTypeO o[CTA_TILE_Q * HEAD_DIM_CKV]; @@ -135,12 +138,12 @@ struct HopperKernelTraits // before WGMMA. Match on the exact type (not sizeof==1) — other 1-byte // JIT dtypes (e.g. __nv_fp4x2_e2m1) have no compatible vec_cast. static constexpr bool USE_KV_REPACK = std::is_same_v; - // FP8 KV swizzle / dequant layout is only correct for the DeepSeek MLA + // FP8 KV swizzle / dequant layout is only correct for the supported MLA // dims; AOT/JIT must not instantiate other sizes. static_assert(!USE_KV_REPACK || HEAD_DIM_CKV_ == 512, "FP8 KV MLA path currently only supports HEAD_DIM_CKV=512"); - static_assert(!USE_KV_REPACK || HEAD_DIM_KPE_ == 64, - "FP8 KV MLA path currently only supports HEAD_DIM_KPE=64"); + static_assert(!USE_KV_REPACK || HEAD_DIM_KPE_ == 0 || HEAD_DIM_KPE_ == 64, + "FP8 KV MLA path currently only supports HEAD_DIM_KPE=0 (no PE) or 64"); // Strides for the BF16 dequant staging buffer and the P buffer, both of // which are DTypeQ-typed regardless of DTypeKV. static constexpr uint32_t UPCAST_STRIDE_CKV_BF16 = HEAD_DIM_CKV_ / upcast_size(); @@ -347,9 +350,11 @@ __device__ __forceinline__ void load_kv(typename KTraits::SharedStorage* smem_st // Each thread reads one 16-byte chunk (16 FP8 elems) and writes two // 16-byte chunks (8 BF16 elems each); see the FP8 dequant idiom in prefill.cuh. template -__device__ __forceinline__ void repack_fp8_kv_to_bf16(typename KTraits::SharedStorage* smem_storage, - const uint32_t stage_idx, float ckv_scale, - float kpe_scale) { +__device__ __forceinline__ void repack_fp8_kv_to_bf16( + typename KTraits::SharedStorage* smem_storage, const uint32_t stage_idx, float ckv_scale, + float kpe_scale, const float* ckv_scale_arr = nullptr, uint32_t row_base = 0, + const typename KTraits::IdType* kv_indices = nullptr, uint_fastdiv block_size = uint_fastdiv(), + uint32_t packed_kv_bound = 0) { using DTypeKV = typename KTraits::DTypeKV; using DTypeQ = typename KTraits::DTypeQ; static_assert(std::is_same_v, @@ -368,14 +373,11 @@ __device__ __forceinline__ void repack_fp8_kv_to_bf16(typename KTraits::SharedSt // b128 column counts (16-byte chunks per row). constexpr uint32_t FP8_COLS_CKV = HEAD_DIM_CKV / upcast_size(); constexpr uint32_t BF16_COLS_CKV = HEAD_DIM_CKV / upcast_size(); - constexpr uint32_t FP8_COLS_KPE = HEAD_DIM_KPE / upcast_size(); constexpr uint32_t BF16_COLS_KPE = HEAD_DIM_KPE / upcast_size(); constexpr uint32_t NUM_B128_CKV = CTA_TILE_KV * FP8_COLS_CKV; - constexpr uint32_t NUM_B128_KPE = CTA_TILE_KV * FP8_COLS_KPE; using packed2_t = std::conditional_t, half2, nv_bfloat162>; packed2_t ckv_scale_packed{static_cast(ckv_scale), static_cast(ckv_scale)}; - packed2_t kpe_scale_packed{static_cast(kpe_scale), static_cast(kpe_scale)}; b128_t* src_ckv = (b128_t*)smem_storage->kv_o_smem[stage_idx].ckv; b128_t* dst_ckv = (b128_t*)smem_storage->ckv_bf16; @@ -387,9 +389,23 @@ __device__ __forceinline__ void repack_fp8_kv_to_bf16(typename KTraits::SharedSt col)]; alignas(16) DTypeQ conv[16]; vec_cast::template cast<16>(conv, (DTypeKV*)&packed); + float scale = ckv_scale; + if (ckv_scale_arr) { + uint32_t page, off; + block_size.divmod(row_base + row, page, off); + uint32_t phys = (row_base + row) < packed_kv_bound ? kv_indices[page] * block_size + off : 0; + scale = ckv_scale_arr[phys * (HEAD_DIM_CKV / 128) + col / (128 / upcast_size())]; + } + if (ckv_scale_arr) { +#pragma unroll + for (uint32_t k = 0; k < 16; ++k) { + conv[k] = static_cast(static_cast(conv[k]) * scale); + } + } else { #pragma unroll - for (uint32_t k = 0; k < 8; ++k) { - ((packed2_t*)&conv[0])[k] = __hmul2(((packed2_t*)&conv[0])[k], ckv_scale_packed); + for (uint32_t k = 0; k < 8; ++k) { + ((packed2_t*)&conv[0])[k] = __hmul2(((packed2_t*)&conv[0])[k], ckv_scale_packed); + } } dst_ckv[get_swizzle_offset( row, 2 * col)] = *(b128_t*)&conv[0]; @@ -397,24 +413,30 @@ __device__ __forceinline__ void repack_fp8_kv_to_bf16(typename KTraits::SharedSt row, 2 * col + 1)] = *(b128_t*)&conv[8]; } - b128_t* src_kpe = (b128_t*)smem_storage->kv_o_smem[stage_idx].kpe; - b128_t* dst_kpe = (b128_t*)smem_storage->kpe_bf16; + if constexpr (HEAD_DIM_KPE > 0) { + constexpr uint32_t FP8_COLS_KPE = HEAD_DIM_KPE / upcast_size(); + constexpr uint32_t NUM_B128_KPE = CTA_TILE_KV * FP8_COLS_KPE; + packed2_t kpe_scale_packed{static_cast(kpe_scale), static_cast(kpe_scale)}; + + b128_t* src_kpe = (b128_t*)smem_storage->kv_o_smem[stage_idx].kpe; + b128_t* dst_kpe = (b128_t*)smem_storage->kpe_bf16; #pragma unroll - for (uint32_t idx = thread_id; idx < NUM_B128_KPE; idx += num_threads) { - uint32_t row = idx / FP8_COLS_KPE, col = idx % FP8_COLS_KPE; - b128_t packed = - src_kpe[get_swizzle_offset(row, - col)]; - alignas(16) DTypeQ conv[16]; - vec_cast::template cast<16>(conv, (DTypeKV*)&packed); + for (uint32_t idx = thread_id; idx < NUM_B128_KPE; idx += num_threads) { + uint32_t row = idx / FP8_COLS_KPE, col = idx % FP8_COLS_KPE; + b128_t packed = + src_kpe[get_swizzle_offset( + row, col)]; + alignas(16) DTypeQ conv[16]; + vec_cast::template cast<16>(conv, (DTypeKV*)&packed); #pragma unroll - for (uint32_t k = 0; k < 8; ++k) { - ((packed2_t*)&conv[0])[k] = __hmul2(((packed2_t*)&conv[0])[k], kpe_scale_packed); + for (uint32_t k = 0; k < 8; ++k) { + ((packed2_t*)&conv[0])[k] = __hmul2(((packed2_t*)&conv[0])[k], kpe_scale_packed); + } + dst_kpe[get_swizzle_offset( + row, 2 * col)] = *(b128_t*)&conv[0]; + dst_kpe[get_swizzle_offset( + row, 2 * col + 1)] = *(b128_t*)&conv[8]; } - dst_kpe[get_swizzle_offset( - row, 2 * col)] = *(b128_t*)&conv[0]; - dst_kpe[get_swizzle_offset( - row, 2 * col + 1)] = *(b128_t*)&conv[8]; } } @@ -1069,8 +1091,12 @@ __global__ __launch_bounds__(KTraits::NUM_THREADS) void BatchMLAPageAttentionHop __syncthreads(); // Only consumer warpgroup (128 threads) dequants FP8 KV -> BF16 // staging buffers. Producer wg passes through the d1/d2/d_end syncs. - repack_fp8_kv_to_bf16(&smem_storage, smem_pipe_read_kv.index(), - variant.ckv_scale, variant.kpe_scale); + repack_fp8_kv_to_bf16( + &smem_storage, smem_pipe_read_kv.index(), variant.ckv_scale, variant.kpe_scale, + variant.ckv_scale_arr, + kv_indptr * block_size + kv_start + + static_cast(kv_tile_idx) * KTraits::CTA_TILE_KV, + kv_indices, block_size, kv_indptr * block_size + kv_len); // s2: ensures dequant is complete before any wg reads BF16 staging. __syncthreads(); } @@ -1114,8 +1140,12 @@ __global__ __launch_bounds__(KTraits::NUM_THREADS) void BatchMLAPageAttentionHop __syncthreads(); // Only consumer warpgroup (128 threads) dequants FP8 KV -> BF16 // staging buffers. Producer wg passes through the d1/d2/d_end syncs. - repack_fp8_kv_to_bf16(&smem_storage, smem_pipe_read_kv.index(), - variant.ckv_scale, variant.kpe_scale); + repack_fp8_kv_to_bf16( + &smem_storage, smem_pipe_read_kv.index(), variant.ckv_scale, variant.kpe_scale, + variant.ckv_scale_arr, + kv_indptr * block_size + kv_start + + static_cast(kv_tile_idx) * KTraits::CTA_TILE_KV, + kv_indices, block_size, kv_indptr * block_size + kv_len); // s2: ensures dequant is complete before any wg reads BF16 staging. __syncthreads(); } @@ -1156,8 +1186,12 @@ __global__ __launch_bounds__(KTraits::NUM_THREADS) void BatchMLAPageAttentionHop __syncthreads(); // Only consumer warpgroup (128 threads) dequants FP8 KV -> BF16 // staging buffers. Producer wg passes through the d1/d2/d_end syncs. - repack_fp8_kv_to_bf16(&smem_storage, smem_pipe_read_kv.index(), - variant.ckv_scale, variant.kpe_scale); + repack_fp8_kv_to_bf16( + &smem_storage, smem_pipe_read_kv.index(), variant.ckv_scale, variant.kpe_scale, + variant.ckv_scale_arr, + kv_indptr * block_size + kv_start + + static_cast(kv_tile_idx) * KTraits::CTA_TILE_KV, + kv_indices, block_size, kv_indptr * block_size + kv_len); // s2: ensures dequant is complete before any wg reads BF16 staging. __syncthreads(); } diff --git a/include/flashinfer/attention/mla_params.cuh b/include/flashinfer/attention/mla_params.cuh index 0e18e7fbaab..4cc30e454b0 100644 --- a/include/flashinfer/attention/mla_params.cuh +++ b/include/flashinfer/attention/mla_params.cuh @@ -76,6 +76,8 @@ struct MLAParams { // dequantized = ckv_scale * static_cast(ckv_fp8); same for kpe. float ckv_scale = 1.f; float kpe_scale = 1.f; + // Row-major [token][128-element group] FP32 scales override ckv_scale. + const float* ckv_scale_arr = nullptr; bool return_lse_base_on_e; }; diff --git a/tests/attention/test_deepseek_mla.py b/tests/attention/test_deepseek_mla.py index d7c14978e78..29a4c228d8f 100644 --- a/tests/attention/test_deepseek_mla.py +++ b/tests/attention/test_deepseek_mla.py @@ -830,6 +830,20 @@ def _ref_dequant_to_bf16(fp8: torch.Tensor, scale: float) -> torch.Tensor: return (fp8.to(torch.bfloat16) * scale_bf16).to(torch.bfloat16) +def _per_group_symmetric_quant_fp8( + x: torch.Tensor, +) -> tuple[torch.Tensor, torch.Tensor]: + grouped = x.float().reshape(*x.shape[:-1], HEAD_DIM_CKV // 128, 128) + scales = (grouped.abs().amax(dim=-1) / 448.0).clamp_min(1e-8) + quantized = (grouped / scales.unsqueeze(-1)).clamp(-448, 448) + return quantized.to(torch.float8_e4m3fn).reshape_as(x), scales.contiguous() + + +def _ref_group_dequant_to_bf16(fp8: torch.Tensor, scales: torch.Tensor) -> torch.Tensor: + grouped = fp8.reshape(*fp8.shape[:-1], HEAD_DIM_CKV // 128, 128) + return (grouped.float() * scales.unsqueeze(-1)).to(torch.bfloat16).reshape_as(fp8) + + def _run_mla( backend: str, q_nope: torch.Tensor, @@ -848,6 +862,8 @@ def _run_mla( kv_dtype: torch.dtype, ckv_scale: float | None = None, kpe_scale: float | None = None, + ckv_scale_arr: torch.Tensor | None = None, + head_dim_kpe: int = HEAD_DIM_KPE, ) -> torch.Tensor: device = q_nope.device workspace = torch.empty(128 * 1024 * 1024, dtype=torch.uint8, device=device) @@ -859,7 +875,7 @@ def _run_mla( kv_len_arr, num_heads=num_heads, head_dim_ckv=HEAD_DIM_CKV, - head_dim_kpe=HEAD_DIM_KPE, + head_dim_kpe=head_dim_kpe, page_size=page_size, causal=causal, sm_scale=sm_scale, @@ -871,9 +887,86 @@ def _run_mla( kwargs["ckv_scale"] = ckv_scale if kpe_scale is not None: kwargs["kpe_scale"] = kpe_scale + if ckv_scale_arr is not None: + kwargs["ckv_scale_arr"] = ckv_scale_arr return wrapper.run(q_nope, q_pe, ckv, kpe, **kwargs) +@pytest.mark.parametrize("backend", ["fa2", "fa3"]) +def test_batch_mla_fp8_nope_group_scales_matches_bf16_reference(backend): + if not is_sm90a_supported(torch.device("cuda")): + pytest.skip("FP8 NoPE MLA requires SM90a") + + torch.manual_seed(2026) + device = torch.device("cuda:0") + batch_size, num_heads, page_size = 2, 16, 16 + num_pages = 6 + q_nope = torch.randn( + batch_size, num_heads, HEAD_DIM_CKV, dtype=torch.bfloat16, device=device + ) + q_pe = torch.empty(batch_size, num_heads, 0, dtype=torch.bfloat16, device=device) + + row_scale = torch.linspace(0.02, 0.2, num_pages * page_size, device=device) + group_scale = torch.tensor([0.5, 1.0, 2.0, 4.0], device=device) + ckv = torch.randn(num_pages, page_size, HEAD_DIM_CKV, device=device) + ckv *= row_scale[:, None].reshape(num_pages, page_size, 1) + ckv *= group_scale.repeat_interleave(128) + ckv_fp8, ckv_scales = _per_group_symmetric_quant_fp8(ckv) + ckv_ref = _ref_group_dequant_to_bf16(ckv_fp8, ckv_scales) + kpe_fp8 = torch.empty( + num_pages, page_size, 0, dtype=torch.float8_e4m3fn, device=device + ) + kpe_ref = torch.empty(num_pages, page_size, 0, dtype=torch.bfloat16, device=device) + + qo_indptr = torch.tensor([0, 1, 2], dtype=torch.int32, device=device) + kv_indptr = torch.tensor([0, 3, 5], dtype=torch.int32, device=device) + kv_indices = torch.tensor([4, 1, 3, 0, 5], dtype=torch.int32, device=device) + kv_len_arr = torch.tensor([45, 29], dtype=torch.int32, device=device) + sm_scale = 1.0 / math.sqrt(HEAD_DIM_CKV) + + o_ref = _run_mla( + backend, + q_nope, + q_pe, + ckv_ref, + kpe_ref, + qo_indptr, + kv_indptr, + kv_indices, + kv_len_arr, + num_heads, + page_size, + False, + sm_scale, + torch.bfloat16, + torch.bfloat16, + head_dim_kpe=0, + ) + o_fp8 = _run_mla( + backend, + q_nope, + q_pe, + ckv_fp8, + kpe_fp8, + qo_indptr, + kv_indptr, + kv_indices, + kv_len_arr, + num_heads, + page_size, + False, + sm_scale, + torch.bfloat16, + torch.float8_e4m3fn, + ckv_scale=1.0, + kpe_scale=1.0, + ckv_scale_arr=ckv_scales, + head_dim_kpe=0, + ) + + torch.testing.assert_close(o_fp8, o_ref, rtol=2e-2, atol=2e-2) + + @pytest.mark.parametrize("batch_size", [1, 4, 16]) @pytest.mark.parametrize("kv_len", [256, 1024, 4096]) @pytest.mark.parametrize("qo_len", [1, 16]) @@ -1328,6 +1421,9 @@ def test_fp8_kv_scales_are_keyword_only(): assert params["kpe_scale"].kind == inspect.Parameter.KEYWORD_ONLY, ( f"kpe_scale kind={params['kpe_scale'].kind}, expected KEYWORD_ONLY" ) + assert params["ckv_scale_arr"].kind == inspect.Parameter.KEYWORD_ONLY, ( + f"ckv_scale_arr kind={params['ckv_scale_arr'].kind}, expected KEYWORD_ONLY" + ) @pytest.mark.parametrize( @@ -1482,6 +1578,19 @@ def test_fp8_kv_requires_scales(): ) with pytest.raises(ValueError, match="ckv_scale and kpe_scale are required"): wrapper.run(q_nope, q_pe, ckv_fp8, kpe_fp8) + invalid_scales = torch.ones( + 1, page_size, HEAD_DIM_CKV // 128 - 1, dtype=torch.float32, device=device + ) + with pytest.raises(ValueError, match="Invalid shape of ckv_scale_arr"): + wrapper.run( + q_nope, + q_pe, + ckv_fp8, + kpe_fp8, + ckv_scale=1.0, + kpe_scale=1.0, + ckv_scale_arr=invalid_scales, + ) if __name__ == "__main__": From 67fb8f37c8997738e74847d8ec7db34cee33bcf9 Mon Sep 17 00:00:00 2001 From: Xinyuan Tong Date: Wed, 12 Aug 2026 16:50:29 +0000 Subject: [PATCH 3/5] fix: address FP8 MLA review findings --- flashinfer/mla/_core.py | 5 +- flashinfer/trace/templates/attention.py | 17 +++++-- include/flashinfer/attention/mla.cuh | 51 +++++++++++-------- include/flashinfer/attention/mla_hopper.cuh | 6 +-- tests/attention/test_deepseek_mla.py | 7 +-- ...mla_paged_decode_h16_ckv512_kpe64_ps1.json | 20 ++++++-- ...la_paged_decode_h16_ckv512_kpe64_ps64.json | 20 ++++++-- tests/trace/test_fi_trace.py | 13 ++++- 8 files changed, 101 insertions(+), 38 deletions(-) diff --git a/flashinfer/mla/_core.py b/flashinfer/mla/_core.py index 46f8073b0ae..25ea4afc294 100644 --- a/flashinfer/mla/_core.py +++ b/flashinfer/mla/_core.py @@ -2520,7 +2520,10 @@ def run( raise ValueError( "ckv_scale_arr is only valid when kv_data_type is FP8." ) - expected_scale_shape = ckv_cache.shape[:-1] + (self._head_dim_ckv // 128,) + expected_scale_shape = ( + *ckv_cache.shape[:-1], + self._head_dim_ckv // 128, + ) check_shape_dtype_device( ckv_scale_arr, expected_scale_shape, diff --git a/flashinfer/trace/templates/attention.py b/flashinfer/trace/templates/attention.py index 2aeb6cfab68..ab8799efff8 100644 --- a/flashinfer/trace/templates/attention.py +++ b/flashinfer/trace/templates/attention.py @@ -1425,6 +1425,7 @@ def _mla_paged_decode_init( num_pages_per_seq: int = 4, len_indptr: int = 0, num_kv_indices: int = 0, + ckv_scale_groups: int = 0, device: str = "cuda", seed: int = 0, ): @@ -1433,7 +1434,7 @@ def _mla_paged_decode_init( Sourced from ``tests/trace/example.py`` MLA section. Default ``num_qo_heads=16`` matches DeepSeek-V3 TP=8. """ - del num_pages, len_indptr, num_kv_indices + del num_pages, len_indptr, num_kv_indices, ckv_scale_groups torch.manual_seed(seed) total_pages = batch_size * num_pages_per_seq qo_indptr = torch.arange(batch_size + 1, dtype=torch.int32, device=device) @@ -1504,6 +1505,7 @@ def _mla_paged_decode_init( "head_dim_ckv": Const(abbrev="ckv"), "head_dim_kpe": Const(abbrev="kpe"), "page_size": Const(abbrev="ps"), + "ckv_scale_groups": Var(description="Number of 128-channel CKV scale groups."), "num_pages": Var( description="Total number of allocated pages in the KV cache." ), @@ -1551,7 +1553,7 @@ def _mla_paged_decode_init( description=( "Per-tensor dequantization scale for the compressed-KV cache when " "kv_data_type is FP8 (real = quantized * ckv_scale). Required " - "together with kpe_scale for the FP8 KV cache path on the fa3 " + "together with kpe_scale for the FP8 KV cache path on the fa2/fa3 " "backend. Set during run(), not plan()." ), ), @@ -1561,10 +1563,19 @@ def _mla_paged_decode_init( description=( "Per-tensor dequantization scale for the rope-K cache when " "kv_data_type is FP8 (real = quantized * kpe_scale). Required " - "together with ckv_scale for the FP8 KV cache path on the fa3 " + "together with ckv_scale for the FP8 KV cache path on the fa2/fa3 " "backend. Set during run(), not plan()." ), ), + "ckv_scale_arr": Tensor( + ["num_pages", "page_size", "ckv_scale_groups"], + dtype="float32", + optional=True, + description=( + "Optional per-token, per-128-channel CKV dequantization scales " + "for the FP8 KV cache path." + ), + ), "return_lse": Scalar( "int32", optional=True, description="Bool: also return LSE." ), diff --git a/include/flashinfer/attention/mla.cuh b/include/flashinfer/attention/mla.cuh index 876dc3ce743..e0048c8c0c3 100644 --- a/include/flashinfer/attention/mla.cuh +++ b/include/flashinfer/attention/mla.cuh @@ -113,6 +113,12 @@ struct KernelTraits { static constexpr uint32_t UPCAST_STRIDE_KPE_BF16 = (HEAD_DIM_KPE > 0 ? HEAD_DIM_KPE : 16) / upcast_size(); static constexpr uint32_t UPCAST_STRIDE_P_BF16 = CTA_TILE_KV / upcast_size(); + static constexpr uint32_t KPE_B128_PER_ROW = HEAD_DIM_KPE * sizeof(DTypeKV_) / 16; + static constexpr uint32_t LANES_PER_ROW_KPE = + KPE_B128_PER_ROW == 0 ? 1 : ((KPE_B128_PER_ROW >= 8) ? 8 : KPE_B128_PER_ROW); + static constexpr uint32_t INNER_LOADS_KPE = KPE_B128_PER_ROW / LANES_PER_ROW_KPE; + static constexpr SwizzleMode SWIZZLE_MODE_KPE_RAW = + (USE_KV_REPACK && KPE_B128_PER_ROW < 8) ? SwizzleMode::k64B : SwizzleMode::k128B; using DTypeQ = DTypeQ_; using DTypeKV = DTypeKV_; @@ -212,7 +218,7 @@ __device__ __forceinline__ void load_kv( const uint32_t warp_idx_in_wg = threadIdx.y; smem_t ckv_smem(smem_storage->ckv_smem[stage_idx]); - smem_t kpe_smem(smem_storage->kpe_p_smem[stage_idx]); + smem_t kpe_smem(smem_storage->kpe_p_smem[stage_idx]); if constexpr (KTraits::NUM_MMA_KV == 1) { if (warpgroup_idx == 0) { @@ -241,13 +247,17 @@ __device__ __forceinline__ void load_kv( ckv_ptr += 8 * upcast_size(); } + if constexpr (KTraits::HEAD_DIM_KPE > 0) { + if (lane_idx % 8 < KTraits::LANES_PER_ROW_KPE) { #pragma unroll - for (uint32_t mma_d = 0; mma_d < KTraits::NUM_MMA_D_KPE / 4; ++mma_d) { - uint32_t kpe_smem_offset_w = kpe_smem.template get_permuted_offset( - warp_idx_in_wg * 4 + lane_idx / 8, 8 * mma_d + lane_idx % 8); - kpe_smem.load_128b_async(kpe_smem_offset_w, kpe_ptr, - packed_block_iter < packed_kv_bound); - kpe_ptr += 8 * upcast_size(); + for (uint32_t mma_d = 0; mma_d < KTraits::INNER_LOADS_KPE; ++mma_d) { + uint32_t kpe_smem_offset_w = kpe_smem.template get_permuted_offset( + warp_idx_in_wg * 4 + lane_idx / 8, 8 * mma_d + lane_idx % 8); + kpe_smem.load_128b_async( + kpe_smem_offset_w, kpe_ptr, packed_block_iter < packed_kv_bound); + kpe_ptr += 8 * upcast_size(); + } + } } } } else { @@ -280,14 +290,18 @@ __device__ __forceinline__ void load_kv( ckv_ptr += 8 * upcast_size(); } + if constexpr (KTraits::HEAD_DIM_KPE > 0) { + if (lane_idx % 8 < KTraits::LANES_PER_ROW_KPE) { #pragma unroll - for (uint32_t mma_d = 0; mma_d < KTraits::NUM_MMA_D_KPE / 4; ++mma_d) { - uint32_t kpe_smem_offset_w = kpe_smem.template get_permuted_offset( - 32 * mma_kv + warpgroup_idx * 16 + warp_idx_in_wg * 4 + lane_idx / 8, - 8 * mma_d + lane_idx % 8); - kpe_smem.load_128b_async(kpe_smem_offset_w, kpe_ptr, - packed_block_iter < packed_kv_bound); - kpe_ptr += 8 * upcast_size(); + for (uint32_t mma_d = 0; mma_d < KTraits::INNER_LOADS_KPE; ++mma_d) { + uint32_t kpe_smem_offset_w = kpe_smem.template get_permuted_offset( + 32 * mma_kv + warpgroup_idx * 16 + warp_idx_in_wg * 4 + lane_idx / 8, + 8 * mma_d + lane_idx % 8); + kpe_smem.load_128b_async( + kpe_smem_offset_w, kpe_ptr, packed_block_iter < packed_kv_bound); + kpe_ptr += 8 * upcast_size(); + } + } } } } @@ -319,22 +333,19 @@ __device__ __forceinline__ void repack_fp8_kv_to_bf16( ckv_src.base[ckv_src.template get_permuted_offset(row, col)]; alignas(16) DTypeQ conv[16]; vec_cast::template cast<16>(conv, (DTypeKV*)&packed); - float scale = ckv_scale; if (ckv_scale_arr != nullptr) { uint32_t page, offset; block_size.divmod(row_base + row, page, offset); uint32_t physical_row = (row_base + row) < packed_kv_bound ? kv_indices[page] * block_size + offset : 0; - scale = + float scale = ckv_scale_arr[physical_row * (HEAD_DIM_CKV / 128) + col / (128 / upcast_size())]; - } - if (ckv_scale_arr != nullptr) { #pragma unroll for (uint32_t k = 0; k < 16; ++k) { conv[k] = static_cast(static_cast(conv[k]) * scale); } } else { - packed2_t scale_packed{static_cast(scale), static_cast(scale)}; + packed2_t scale_packed{static_cast(ckv_scale), static_cast(ckv_scale)}; #pragma unroll for (uint32_t k = 0; k < 8; ++k) { ((packed2_t*)&conv[0])[k] = __hmul2(((packed2_t*)&conv[0])[k], scale_packed); @@ -351,7 +362,7 @@ __device__ __forceinline__ void repack_fp8_kv_to_bf16( constexpr uint32_t FP8_COLS_KPE = HEAD_DIM_KPE / upcast_size(); constexpr uint32_t NUM_B128_KPE = CTA_TILE_KV * FP8_COLS_KPE; packed2_t kpe_scale_packed{static_cast(kpe_scale), static_cast(kpe_scale)}; - smem_t kpe_src(smem_storage->kpe_p_smem[stage_idx]); + smem_t kpe_src(smem_storage->kpe_p_smem[stage_idx]); smem_t kpe_dst(smem_storage->kpe_bf16 + stage_idx * CTA_TILE_KV * HEAD_DIM_KPE); #pragma unroll diff --git a/include/flashinfer/attention/mla_hopper.cuh b/include/flashinfer/attention/mla_hopper.cuh index 85a5f19b734..d8baa8cf3d8 100644 --- a/include/flashinfer/attention/mla_hopper.cuh +++ b/include/flashinfer/attention/mla_hopper.cuh @@ -389,14 +389,12 @@ __device__ __forceinline__ void repack_fp8_kv_to_bf16( col)]; alignas(16) DTypeQ conv[16]; vec_cast::template cast<16>(conv, (DTypeKV*)&packed); - float scale = ckv_scale; if (ckv_scale_arr) { uint32_t page, off; block_size.divmod(row_base + row, page, off); uint32_t phys = (row_base + row) < packed_kv_bound ? kv_indices[page] * block_size + off : 0; - scale = ckv_scale_arr[phys * (HEAD_DIM_CKV / 128) + col / (128 / upcast_size())]; - } - if (ckv_scale_arr) { + float scale = + ckv_scale_arr[phys * (HEAD_DIM_CKV / 128) + col / (128 / upcast_size())]; #pragma unroll for (uint32_t k = 0; k < 16; ++k) { conv[k] = static_cast(static_cast(conv[k]) * scale); diff --git a/tests/attention/test_deepseek_mla.py b/tests/attention/test_deepseek_mla.py index 29a4c228d8f..68581b5c7cb 100644 --- a/tests/attention/test_deepseek_mla.py +++ b/tests/attention/test_deepseek_mla.py @@ -1283,7 +1283,8 @@ def test_batch_mla_fp8_kv_zero_kv_gives_zero_output(): assert o.abs().max().item() == 0.0, f"non-zero output: {o.abs().max().item()}" -def test_fp8_kv_kpe_dominant_no_row_aliasing(): +@pytest.mark.parametrize("backend", ["fa2", "fa3"]) +def test_fp8_kv_kpe_dominant_no_row_aliasing(backend): """Deterministic regression for the FP8 KPE shmem swizzle aliasing bug. With HEAD_DIM_KPE=64 on the FP8 path, the raw KPE buffer has 4 b128 @@ -1327,7 +1328,7 @@ def test_fp8_kv_kpe_dominant_no_row_aliasing(): kv_indices = torch.arange(0, nps, dtype=torch.int32, device=device) o_bf16 = _run_mla( - "fa3", + backend, q_nope, q_pe, ckv_ref, @@ -1344,7 +1345,7 @@ def test_fp8_kv_kpe_dominant_no_row_aliasing(): kv_dtype=torch.bfloat16, ) o_fp8 = _run_mla( - "fa3", + backend, q_nope, q_pe, ckv_fp8, diff --git a/tests/trace/fi_trace_out/mla_paged_decode_h16_ckv512_kpe64_ps1.json b/tests/trace/fi_trace_out/mla_paged_decode_h16_ckv512_kpe64_ps1.json index a1a206e5fd6..a6f3ad7ec55 100644 --- a/tests/trace/fi_trace_out/mla_paged_decode_h16_ckv512_kpe64_ps1.json +++ b/tests/trace/fi_trace_out/mla_paged_decode_h16_ckv512_kpe64_ps1.json @@ -28,6 +28,10 @@ "type": "const", "value": 1 }, + "ckv_scale_groups": { + "type": "var", + "description": "Number of 128-channel CKV scale groups." + }, "num_pages": { "type": "var", "description": "Total number of allocated pages in the KV cache." @@ -108,13 +112,23 @@ "shape": null, "dtype": "float32", "optional": true, - "description": "Per-tensor dequantization scale for the compressed-KV cache when kv_data_type is FP8 (real = quantized * ckv_scale). Required together with kpe_scale for the FP8 KV cache path on the fa3 backend. Set during run(), not plan()." + "description": "Per-tensor dequantization scale for the compressed-KV cache when kv_data_type is FP8 (real = quantized * ckv_scale). Required together with kpe_scale for the FP8 KV cache path on the fa2/fa3 backend. Set during run(), not plan()." }, "kpe_scale": { "shape": null, "dtype": "float32", "optional": true, - "description": "Per-tensor dequantization scale for the rope-K cache when kv_data_type is FP8 (real = quantized * kpe_scale). Required together with ckv_scale for the FP8 KV cache path on the fa3 backend. Set during run(), not plan()." + "description": "Per-tensor dequantization scale for the rope-K cache when kv_data_type is FP8 (real = quantized * kpe_scale). Required together with ckv_scale for the FP8 KV cache path on the fa2/fa3 backend. Set during run(), not plan()." + }, + "ckv_scale_arr": { + "shape": [ + "num_pages", + "page_size", + "ckv_scale_groups" + ], + "dtype": "float32", + "optional": true, + "description": "Optional per-token, per-128-channel CKV dequantization scales for the FP8 KV cache path." }, "return_lse": { "shape": null, @@ -144,5 +158,5 @@ }, "reference": "from __future__ import annotations\nimport math\nimport torch\nimport torch.nn.functional as F\n\n@torch.no_grad()\ndef _mla_paged_decode_reference(\n q_nope, q_pe, ckv_cache, kpe_cache, kv_indptr, kv_indices, sm_scale\n):\n batch_size, num_qo_heads, head_dim_ckv = q_nope.shape\n _, _, head_dim_kpe = q_pe.shape\n\n # [num_pages, page_size, head_dim_*] \u2014 keep the page dim; flatten after gather.\n Kc_all = ckv_cache.to(torch.float32)\n Kp_all = kpe_cache.to(torch.float32)\n\n output = torch.zeros(\n (batch_size, num_qo_heads, head_dim_ckv),\n dtype=torch.bfloat16,\n device=q_nope.device,\n )\n lse = torch.full(\n (batch_size, num_qo_heads),\n -float(\"inf\"),\n dtype=torch.float32,\n device=q_nope.device,\n )\n\n for b in range(batch_size):\n page_beg = int(kv_indptr[b].item())\n page_end = int(kv_indptr[b + 1].item())\n if page_beg >= page_end:\n output[b].zero_()\n continue\n # kv_indices are page IDs; gather pages then flatten to a token axis.\n page_ids = kv_indices[page_beg:page_end].to(torch.long)\n Kc = Kc_all[page_ids].reshape(-1, head_dim_ckv) # [L, head_dim_ckv]\n Kp = Kp_all[page_ids].reshape(-1, head_dim_kpe) # [L, head_dim_kpe]\n qn = q_nope[b].to(torch.float32) # [num_qo_heads, head_dim_ckv]\n qp = q_pe[b].to(torch.float32) # [num_qo_heads, head_dim_kpe]\n logits = ((qn @ Kc.T) + (qp @ Kp.T)) * sm_scale # [num_qo_heads, L]\n lse[b] = torch.logsumexp(logits, dim=-1) / math.log(2.0)\n output[b] = (torch.softmax(logits, dim=-1) @ Kc).to(torch.bfloat16)\n\n return output, lse\n", "check": "def standard_check(\n reference_outputs: Any,\n actual_outputs: Any,\n *,\n rtol: Optional[float] = None,\n atol: Optional[float] = None,\n max_mismatch_pct: float = 0.0,\n min_cos_sim: Optional[float] = 1.0 - 1e-3,\n) -> bool:\n \"\"\"Default trace correctness check used when a template does not override it.\"\"\"\n from flashinfer.trace import default_check\n\n return default_check(\n reference_outputs,\n actual_outputs,\n rtol=rtol,\n atol=atol,\n max_mismatch_pct=max_mismatch_pct,\n min_cos_sim=min_cos_sim,\n )\n", - "init": "from __future__ import annotations\nimport math\nimport torch\n\n# ----- shared init helpers -----\n# Copyright (c) 2025 by FlashInfer team.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\n\"\"\"Shared helpers used by ``TraceTemplate.init`` functions.\n\nThis module contains the small set of input-construction patterns that\nrecur across many templates (paged-KV cache index arrays, ragged indptr,\nRoPE pos_ids and cos/sin caches, sampling probs). Each helper is short and\ndocumented; init functions in ``templates/.py`` call into here so\nthe per-template init bodies stay focused on shape/dtype, not boilerplate.\n\nThe full source of this module is **inlined into every dumped JSON's\n``\"init\"`` field** by ``flashinfer/trace/template.py:_render_init_source``,\nso downstream consumers don't need flashinfer installed to re-run the init\nsnippets.\n\"\"\"\n\n\nfrom typing import Optional, Tuple\n\nimport torch\n\n\ndef make_paged_kv_indices(\n batch_size: int,\n num_pages_per_seq: int,\n page_size: int,\n *,\n device: str = \"cuda\",\n) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor]:\n \"\"\"Return ``(kv_indptr, kv_indices, kv_last_page_len)`` for a uniform batch.\n\n Every sequence is assigned exactly ``num_pages_per_seq`` pages, fully\n populated (last-page length == page_size).\n\n Invariants\n ----------\n - ``kv_indptr.shape == (batch_size + 1,)``, dtype int32, monotonic, [0]=0.\n - ``kv_indices == arange(0, batch_size * num_pages_per_seq)``, int32.\n - ``kv_last_page_len == full(batch_size, page_size)``, int32.\n \"\"\"\n total_pages = batch_size * num_pages_per_seq\n kv_indptr = (\n torch.arange(batch_size + 1, dtype=torch.int32, device=device)\n * num_pages_per_seq\n )\n kv_indices = torch.arange(total_pages, dtype=torch.int32, device=device)\n kv_last_page_len = torch.full(\n (batch_size,), page_size, dtype=torch.int32, device=device\n )\n return kv_indptr, kv_indices, kv_last_page_len\n\n\ndef make_ragged_indptr(\n seg_lens,\n *,\n device: str = \"cuda\",\n dtype: torch.dtype = torch.int32,\n) -> torch.Tensor:\n \"\"\"Return cumulative-sum ``indptr`` of length ``len(seg_lens)+1``.\n\n ``seg_lens`` may be a list / tuple / 1-D tensor of segment lengths.\n \"\"\"\n if isinstance(seg_lens, torch.Tensor):\n lens = seg_lens.to(device=device, dtype=dtype)\n else:\n lens = torch.tensor(list(seg_lens), dtype=dtype, device=device)\n indptr = torch.zeros(lens.numel() + 1, dtype=dtype, device=device)\n indptr[1:] = torch.cumsum(lens, dim=0).to(dtype)\n return indptr\n\n\ndef make_uniform_qo_indptr(\n batch_size: int,\n qo_len: int,\n *,\n device: str = \"cuda\",\n) -> torch.Tensor:\n \"\"\"Return ``[0, qo_len, 2*qo_len, ..., batch_size*qo_len]`` int32.\"\"\"\n return torch.arange(batch_size + 1, dtype=torch.int32, device=device) * qo_len\n\n\ndef make_pos_ids(\n nnz: int,\n max_seq_len: Optional[int] = None,\n *,\n device: str = \"cuda\",\n) -> torch.Tensor:\n \"\"\"Return ``[0, 1, ..., nnz-1] (% max_seq_len)`` as int32 on ``device``.\n\n If ``max_seq_len`` is None, no wrapping is applied.\n \"\"\"\n pos = torch.arange(nnz, dtype=torch.int32, device=device)\n if max_seq_len is not None:\n pos = pos % max_seq_len\n return pos\n\n\ndef make_rope_cos_sin_cache(\n max_seq_len: int,\n rope_dim: int,\n *,\n base: float = 1e4,\n device: str = \"cuda\",\n dtype: torch.dtype = torch.float32,\n) -> torch.Tensor:\n \"\"\"Return concatenated ``[cos | sin]`` cache of shape ``[max_seq_len, rope_dim]``.\"\"\"\n t = torch.arange(max_seq_len, dtype=torch.float32, device=device)\n inv = 1.0 / (\n base\n ** (torch.arange(0, rope_dim, 2, dtype=torch.float32, device=device) / rope_dim)\n )\n freqs = t.unsqueeze(-1) * inv.unsqueeze(0)\n cache = torch.cat([torch.cos(freqs), torch.sin(freqs)], dim=-1)\n return cache.to(dtype)\n\n\ndef make_probs(\n batch_size: int,\n vocab_size: int,\n *,\n device: str = \"cuda\",\n dtype: torch.dtype = torch.float32,\n) -> torch.Tensor:\n \"\"\"Return a ``[batch_size, vocab_size]`` probability distribution.\n\n Uses ``softmax(randn(...))`` so each row sums to 1.0. This mirrors the\n pattern used throughout ``tests/utils/test_sampling.py``.\n \"\"\"\n return torch.softmax(\n torch.randn(batch_size, vocab_size, dtype=torch.float32, device=device),\n dim=-1,\n ).to(dtype)\n\n\ndef make_logits(\n batch_size: int,\n vocab_size: int,\n *,\n device: str = \"cuda\",\n dtype: torch.dtype = torch.float32,\n) -> torch.Tensor:\n \"\"\"Return ``randn(batch_size, vocab_size)`` logits.\"\"\"\n return torch.randn(batch_size, vocab_size, dtype=dtype, device=device)\n\n\ndef fp8_safe_randn(\n *shape: int,\n scale: float = 0.1,\n device: str = \"cuda\",\n dtype: torch.dtype = torch.bfloat16,\n) -> torch.Tensor:\n \"\"\"``randn(*shape) * scale`` \u2014 keeps values in the FP8/FP4 representable range.\n\n Tests for fp8/fp4 paths typically multiply ``randn`` by 0.1 to avoid\n saturation when quantizing. Use this helper to mirror that convention.\n \"\"\"\n return (torch.randn(*shape, dtype=dtype, device=device) * scale).to(dtype)\n\n\ndef per_tensor_fp8_quantize(\n x: torch.Tensor,\n *,\n fp8_dtype: torch.dtype = torch.float8_e4m3fn,\n) -> Tuple[torch.Tensor, torch.Tensor]:\n \"\"\"Per-tensor FP8 quantization, mirroring ``tests/utils_fp8.py:to_float8``.\n\n Returns ``(x_fp8, inv_scale)`` where ``inv_scale`` is the dequant\n multiplier (``float \u2248 fp8 * inv_scale``).\n \"\"\"\n finfo = torch.finfo(fp8_dtype)\n amax = x.abs().amax().clamp(min=1e-12)\n scale = finfo.max / amax\n x_q = (x.float() * scale).clamp(min=finfo.min, max=finfo.max).to(fp8_dtype)\n return x_q, scale.float().reciprocal()\n\n\ndef fp8_block_quant_1d(\n x_bf16: torch.Tensor,\n block: int = 128,\n) -> Tuple[torch.Tensor, torch.Tensor]:\n \"\"\"Quantize ``[T, H]`` activations into FP8 with per-``(token, block)``\n column-block scales. Returns ``(x_fp8, scales)`` where\n ``scales`` has shape ``[T, H // block]``.\n\n Mirrors ``_fp8_block_quant_1d`` in\n ``tests/moe/test_dpsk_fused_moe_fp8.py``.\n \"\"\"\n assert x_bf16.dim() == 2\n T, H = x_bf16.shape\n assert H % block == 0\n nb = H // block\n finfo = torch.finfo(torch.float8_e4m3fn)\n max_fp8 = finfo.max\n x_f32 = x_bf16.to(torch.float32)\n x_fp8 = torch.empty((T, H), dtype=torch.float8_e4m3fn, device=x_bf16.device)\n scales = torch.empty((T, nb), dtype=torch.float32, device=x_bf16.device)\n for j in range(nb):\n sl = slice(j * block, (j + 1) * block)\n blk = x_f32[:, sl]\n amax = torch.amax(torch.abs(blk), dim=1)\n s = torch.where(amax > 0, amax / max_fp8, torch.ones_like(amax))\n x_fp8[:, sl] = (blk / s.unsqueeze(1)).to(torch.float8_e4m3fn)\n scales[:, j] = s\n return x_fp8, scales\n\n\ndef fp8_block_quant_2d(\n w_bf16: torch.Tensor,\n block: int = 128,\n) -> Tuple[torch.Tensor, torch.Tensor]:\n \"\"\"Quantize weights ``[..., R, C]`` with 2-D ``block \u00d7 block`` scales.\n\n Returns ``(w_fp8, scales)`` where ``scales`` has shape\n ``[..., R // block, C // block]``. Mirrors ``_fp8_block_quant_2d`` in\n ``tests/moe/test_dpsk_fused_moe_fp8.py``.\n \"\"\"\n assert w_bf16.dim() >= 2\n *prefix, R, C = w_bf16.shape\n assert R % block == 0 and C % block == 0\n nb_r, nb_c = R // block, C // block\n finfo = torch.finfo(torch.float8_e4m3fn)\n max_fp8 = finfo.max\n w_f32 = w_bf16.to(torch.float32).contiguous()\n prefix_ndim = len(prefix)\n reshaped = w_f32.reshape(*prefix, nb_r, block, nb_c, block)\n permute_dims = tuple(range(prefix_ndim)) + (\n prefix_ndim,\n prefix_ndim + 2,\n prefix_ndim + 1,\n prefix_ndim + 3,\n )\n blocks = reshaped.permute(permute_dims).contiguous()\n amax = torch.amax(torch.abs(blocks), dim=(-1, -2))\n scales = torch.where(\n amax > 0, amax / max_fp8, torch.ones_like(amax, dtype=torch.float32)\n )\n q_blocks = (blocks / scales.unsqueeze(-1).unsqueeze(-1)).to(torch.float8_e4m3fn)\n inv_permute = [0] * (prefix_ndim + 4)\n for i, p in enumerate(permute_dims):\n inv_permute[p] = i\n w_fp8 = q_blocks.permute(*inv_permute).reshape(*prefix, R, C).contiguous()\n return w_fp8, scales\n\n\n__all__ = [\n \"make_paged_kv_indices\",\n \"make_ragged_indptr\",\n \"make_uniform_qo_indptr\",\n \"make_pos_ids\",\n \"make_rope_cos_sin_cache\",\n \"make_probs\",\n \"make_logits\",\n \"fp8_safe_randn\",\n \"per_tensor_fp8_quantize\",\n \"fp8_block_quant_1d\",\n \"fp8_block_quant_2d\",\n]\n\n# ----- init -----\ndef _mla_paged_decode_init(\n *,\n batch_size: int,\n num_qo_heads: int = 16,\n head_dim_ckv: int = 512,\n head_dim_kpe: int = 64,\n page_size: int = 64,\n num_pages: int = 0,\n num_pages_per_seq: int = 4,\n len_indptr: int = 0,\n num_kv_indices: int = 0,\n device: str = \"cuda\",\n seed: int = 0,\n):\n \"\"\"Build inputs for ``BatchMLAPagedAttentionWrapper.run()``.\n\n Sourced from ``tests/trace/example.py`` MLA section. Default\n ``num_qo_heads=16`` matches DeepSeek-V3 TP=8.\n \"\"\"\n del num_pages, len_indptr, num_kv_indices\n torch.manual_seed(seed)\n total_pages = batch_size * num_pages_per_seq\n qo_indptr = torch.arange(batch_size + 1, dtype=torch.int32, device=device)\n kv_indptr, kv_indices, _ = make_paged_kv_indices(\n batch_size, num_pages_per_seq, page_size, device=device\n )\n kv_len = torch.full(\n (batch_size,),\n num_pages_per_seq * page_size,\n dtype=torch.int32,\n device=device,\n )\n q_nope = torch.randn(\n batch_size, num_qo_heads, head_dim_ckv, dtype=torch.bfloat16, device=device\n )\n q_pe = torch.randn(\n batch_size, num_qo_heads, head_dim_kpe, dtype=torch.bfloat16, device=device\n )\n ckv_cache = torch.randn(\n total_pages, page_size, head_dim_ckv, dtype=torch.bfloat16, device=device\n )\n kpe_cache = torch.randn(\n total_pages, page_size, head_dim_kpe, dtype=torch.bfloat16, device=device\n )\n return {\n # plan() signature: qo_indptr, kv_indptr, kv_indices, kv_len_arr,\n # num_heads, head_dim_ckv, head_dim_kpe, page_size, causal, ...\n \"plan\": {\n \"qo_indptr\": qo_indptr,\n \"kv_indptr\": kv_indptr,\n \"kv_indices\": kv_indices,\n \"kv_len_arr\": kv_len,\n \"num_heads\": int(num_qo_heads),\n \"head_dim_ckv\": int(head_dim_ckv),\n \"head_dim_kpe\": int(head_dim_kpe),\n \"page_size\": int(page_size),\n \"causal\": False,\n # tests/attention/test_deepseek_mla.py scales by the pre-absorption\n # QK dimension: qk_nope_head_dim(128) + head_dim_kpe.\n \"sm_scale\": 1.0 / ((128 + head_dim_kpe) ** 0.5),\n \"q_data_type\": torch.bfloat16,\n \"kv_data_type\": torch.bfloat16,\n },\n \"run\": {\n \"q_nope\": q_nope,\n \"q_pe\": q_pe,\n \"ckv_cache\": ckv_cache,\n \"kpe_cache\": kpe_cache,\n },\n }\n" + "init": "from __future__ import annotations\nimport math\nimport torch\n\n# ----- shared init helpers -----\n# Copyright (c) 2025 by FlashInfer team.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\n\"\"\"Shared helpers used by ``TraceTemplate.init`` functions.\n\nThis module contains the small set of input-construction patterns that\nrecur across many templates (paged-KV cache index arrays, ragged indptr,\nRoPE pos_ids and cos/sin caches, sampling probs). Each helper is short and\ndocumented; init functions in ``templates/.py`` call into here so\nthe per-template init bodies stay focused on shape/dtype, not boilerplate.\n\nThe full source of this module is **inlined into every dumped JSON's\n``\"init\"`` field** by ``flashinfer/trace/template.py:_render_init_source``,\nso downstream consumers don't need flashinfer installed to re-run the init\nsnippets.\n\"\"\"\n\n\nfrom typing import Optional, Tuple\n\nimport torch\n\n\ndef make_paged_kv_indices(\n batch_size: int,\n num_pages_per_seq: int,\n page_size: int,\n *,\n device: str = \"cuda\",\n) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor]:\n \"\"\"Return ``(kv_indptr, kv_indices, kv_last_page_len)`` for a uniform batch.\n\n Every sequence is assigned exactly ``num_pages_per_seq`` pages, fully\n populated (last-page length == page_size).\n\n Invariants\n ----------\n - ``kv_indptr.shape == (batch_size + 1,)``, dtype int32, monotonic, [0]=0.\n - ``kv_indices == arange(0, batch_size * num_pages_per_seq)``, int32.\n - ``kv_last_page_len == full(batch_size, page_size)``, int32.\n \"\"\"\n total_pages = batch_size * num_pages_per_seq\n kv_indptr = (\n torch.arange(batch_size + 1, dtype=torch.int32, device=device)\n * num_pages_per_seq\n )\n kv_indices = torch.arange(total_pages, dtype=torch.int32, device=device)\n kv_last_page_len = torch.full(\n (batch_size,), page_size, dtype=torch.int32, device=device\n )\n return kv_indptr, kv_indices, kv_last_page_len\n\n\ndef make_ragged_indptr(\n seg_lens,\n *,\n device: str = \"cuda\",\n dtype: torch.dtype = torch.int32,\n) -> torch.Tensor:\n \"\"\"Return cumulative-sum ``indptr`` of length ``len(seg_lens)+1``.\n\n ``seg_lens`` may be a list / tuple / 1-D tensor of segment lengths.\n \"\"\"\n if isinstance(seg_lens, torch.Tensor):\n lens = seg_lens.to(device=device, dtype=dtype)\n else:\n lens = torch.tensor(list(seg_lens), dtype=dtype, device=device)\n indptr = torch.zeros(lens.numel() + 1, dtype=dtype, device=device)\n indptr[1:] = torch.cumsum(lens, dim=0).to(dtype)\n return indptr\n\n\ndef make_uniform_qo_indptr(\n batch_size: int,\n qo_len: int,\n *,\n device: str = \"cuda\",\n) -> torch.Tensor:\n \"\"\"Return ``[0, qo_len, 2*qo_len, ..., batch_size*qo_len]`` int32.\"\"\"\n return torch.arange(batch_size + 1, dtype=torch.int32, device=device) * qo_len\n\n\ndef make_pos_ids(\n nnz: int,\n max_seq_len: Optional[int] = None,\n *,\n device: str = \"cuda\",\n) -> torch.Tensor:\n \"\"\"Return ``[0, 1, ..., nnz-1] (% max_seq_len)`` as int32 on ``device``.\n\n If ``max_seq_len`` is None, no wrapping is applied.\n \"\"\"\n pos = torch.arange(nnz, dtype=torch.int32, device=device)\n if max_seq_len is not None:\n pos = pos % max_seq_len\n return pos\n\n\ndef make_rope_cos_sin_cache(\n max_seq_len: int,\n rope_dim: int,\n *,\n base: float = 1e4,\n device: str = \"cuda\",\n dtype: torch.dtype = torch.float32,\n) -> torch.Tensor:\n \"\"\"Return concatenated ``[cos | sin]`` cache of shape ``[max_seq_len, rope_dim]``.\"\"\"\n t = torch.arange(max_seq_len, dtype=torch.float32, device=device)\n inv = 1.0 / (\n base\n ** (torch.arange(0, rope_dim, 2, dtype=torch.float32, device=device) / rope_dim)\n )\n freqs = t.unsqueeze(-1) * inv.unsqueeze(0)\n cache = torch.cat([torch.cos(freqs), torch.sin(freqs)], dim=-1)\n return cache.to(dtype)\n\n\ndef make_probs(\n batch_size: int,\n vocab_size: int,\n *,\n device: str = \"cuda\",\n dtype: torch.dtype = torch.float32,\n) -> torch.Tensor:\n \"\"\"Return a ``[batch_size, vocab_size]`` probability distribution.\n\n Uses ``softmax(randn(...))`` so each row sums to 1.0. This mirrors the\n pattern used throughout ``tests/utils/test_sampling.py``.\n \"\"\"\n return torch.softmax(\n torch.randn(batch_size, vocab_size, dtype=torch.float32, device=device),\n dim=-1,\n ).to(dtype)\n\n\ndef make_logits(\n batch_size: int,\n vocab_size: int,\n *,\n device: str = \"cuda\",\n dtype: torch.dtype = torch.float32,\n) -> torch.Tensor:\n \"\"\"Return ``randn(batch_size, vocab_size)`` logits.\"\"\"\n return torch.randn(batch_size, vocab_size, dtype=dtype, device=device)\n\n\ndef fp8_safe_randn(\n *shape: int,\n scale: float = 0.1,\n device: str = \"cuda\",\n dtype: torch.dtype = torch.bfloat16,\n) -> torch.Tensor:\n \"\"\"``randn(*shape) * scale`` \u2014 keeps values in the FP8/FP4 representable range.\n\n Tests for fp8/fp4 paths typically multiply ``randn`` by 0.1 to avoid\n saturation when quantizing. Use this helper to mirror that convention.\n \"\"\"\n return (torch.randn(*shape, dtype=dtype, device=device) * scale).to(dtype)\n\n\ndef per_tensor_fp8_quantize(\n x: torch.Tensor,\n *,\n fp8_dtype: torch.dtype = torch.float8_e4m3fn,\n) -> Tuple[torch.Tensor, torch.Tensor]:\n \"\"\"Per-tensor FP8 quantization, mirroring ``tests/utils_fp8.py:to_float8``.\n\n Returns ``(x_fp8, inv_scale)`` where ``inv_scale`` is the dequant\n multiplier (``float \u2248 fp8 * inv_scale``).\n \"\"\"\n finfo = torch.finfo(fp8_dtype)\n amax = x.abs().amax().clamp(min=1e-12)\n scale = finfo.max / amax\n x_q = (x.float() * scale).clamp(min=finfo.min, max=finfo.max).to(fp8_dtype)\n return x_q, scale.float().reciprocal()\n\n\ndef fp8_block_quant_1d(\n x_bf16: torch.Tensor,\n block: int = 128,\n) -> Tuple[torch.Tensor, torch.Tensor]:\n \"\"\"Quantize ``[T, H]`` activations into FP8 with per-``(token, block)``\n column-block scales. Returns ``(x_fp8, scales)`` where\n ``scales`` has shape ``[T, H // block]``.\n\n Mirrors ``_fp8_block_quant_1d`` in\n ``tests/moe/test_dpsk_fused_moe_fp8.py``.\n \"\"\"\n assert x_bf16.dim() == 2\n T, H = x_bf16.shape\n assert H % block == 0\n nb = H // block\n finfo = torch.finfo(torch.float8_e4m3fn)\n max_fp8 = finfo.max\n x_f32 = x_bf16.to(torch.float32)\n x_fp8 = torch.empty((T, H), dtype=torch.float8_e4m3fn, device=x_bf16.device)\n scales = torch.empty((T, nb), dtype=torch.float32, device=x_bf16.device)\n for j in range(nb):\n sl = slice(j * block, (j + 1) * block)\n blk = x_f32[:, sl]\n amax = torch.amax(torch.abs(blk), dim=1)\n s = torch.where(amax > 0, amax / max_fp8, torch.ones_like(amax))\n x_fp8[:, sl] = (blk / s.unsqueeze(1)).to(torch.float8_e4m3fn)\n scales[:, j] = s\n return x_fp8, scales\n\n\ndef fp8_block_quant_2d(\n w_bf16: torch.Tensor,\n block: int = 128,\n) -> Tuple[torch.Tensor, torch.Tensor]:\n \"\"\"Quantize weights ``[..., R, C]`` with 2-D ``block \u00d7 block`` scales.\n\n Returns ``(w_fp8, scales)`` where ``scales`` has shape\n ``[..., R // block, C // block]``. Mirrors ``_fp8_block_quant_2d`` in\n ``tests/moe/test_dpsk_fused_moe_fp8.py``.\n \"\"\"\n assert w_bf16.dim() >= 2\n *prefix, R, C = w_bf16.shape\n assert R % block == 0 and C % block == 0\n nb_r, nb_c = R // block, C // block\n finfo = torch.finfo(torch.float8_e4m3fn)\n max_fp8 = finfo.max\n w_f32 = w_bf16.to(torch.float32).contiguous()\n prefix_ndim = len(prefix)\n reshaped = w_f32.reshape(*prefix, nb_r, block, nb_c, block)\n permute_dims = tuple(range(prefix_ndim)) + (\n prefix_ndim,\n prefix_ndim + 2,\n prefix_ndim + 1,\n prefix_ndim + 3,\n )\n blocks = reshaped.permute(permute_dims).contiguous()\n amax = torch.amax(torch.abs(blocks), dim=(-1, -2))\n scales = torch.where(\n amax > 0, amax / max_fp8, torch.ones_like(amax, dtype=torch.float32)\n )\n q_blocks = (blocks / scales.unsqueeze(-1).unsqueeze(-1)).to(torch.float8_e4m3fn)\n inv_permute = [0] * (prefix_ndim + 4)\n for i, p in enumerate(permute_dims):\n inv_permute[p] = i\n w_fp8 = q_blocks.permute(*inv_permute).reshape(*prefix, R, C).contiguous()\n return w_fp8, scales\n\n\n__all__ = [\n \"make_paged_kv_indices\",\n \"make_ragged_indptr\",\n \"make_uniform_qo_indptr\",\n \"make_pos_ids\",\n \"make_rope_cos_sin_cache\",\n \"make_probs\",\n \"make_logits\",\n \"fp8_safe_randn\",\n \"per_tensor_fp8_quantize\",\n \"fp8_block_quant_1d\",\n \"fp8_block_quant_2d\",\n]\n\n# ----- init -----\ndef _mla_paged_decode_init(\n *,\n batch_size: int,\n num_qo_heads: int = 16,\n head_dim_ckv: int = 512,\n head_dim_kpe: int = 64,\n page_size: int = 64,\n num_pages: int = 0,\n num_pages_per_seq: int = 4,\n len_indptr: int = 0,\n num_kv_indices: int = 0,\n ckv_scale_groups: int = 0,\n device: str = \"cuda\",\n seed: int = 0,\n):\n \"\"\"Build inputs for ``BatchMLAPagedAttentionWrapper.run()``.\n\n Sourced from ``tests/trace/example.py`` MLA section. Default\n ``num_qo_heads=16`` matches DeepSeek-V3 TP=8.\n \"\"\"\n del num_pages, len_indptr, num_kv_indices, ckv_scale_groups\n torch.manual_seed(seed)\n total_pages = batch_size * num_pages_per_seq\n qo_indptr = torch.arange(batch_size + 1, dtype=torch.int32, device=device)\n kv_indptr, kv_indices, _ = make_paged_kv_indices(\n batch_size, num_pages_per_seq, page_size, device=device\n )\n kv_len = torch.full(\n (batch_size,),\n num_pages_per_seq * page_size,\n dtype=torch.int32,\n device=device,\n )\n q_nope = torch.randn(\n batch_size, num_qo_heads, head_dim_ckv, dtype=torch.bfloat16, device=device\n )\n q_pe = torch.randn(\n batch_size, num_qo_heads, head_dim_kpe, dtype=torch.bfloat16, device=device\n )\n ckv_cache = torch.randn(\n total_pages, page_size, head_dim_ckv, dtype=torch.bfloat16, device=device\n )\n kpe_cache = torch.randn(\n total_pages, page_size, head_dim_kpe, dtype=torch.bfloat16, device=device\n )\n return {\n # plan() signature: qo_indptr, kv_indptr, kv_indices, kv_len_arr,\n # num_heads, head_dim_ckv, head_dim_kpe, page_size, causal, ...\n \"plan\": {\n \"qo_indptr\": qo_indptr,\n \"kv_indptr\": kv_indptr,\n \"kv_indices\": kv_indices,\n \"kv_len_arr\": kv_len,\n \"num_heads\": int(num_qo_heads),\n \"head_dim_ckv\": int(head_dim_ckv),\n \"head_dim_kpe\": int(head_dim_kpe),\n \"page_size\": int(page_size),\n \"causal\": False,\n # tests/attention/test_deepseek_mla.py scales by the pre-absorption\n # QK dimension: qk_nope_head_dim(128) + head_dim_kpe.\n \"sm_scale\": 1.0 / ((128 + head_dim_kpe) ** 0.5),\n \"q_data_type\": torch.bfloat16,\n \"kv_data_type\": torch.bfloat16,\n },\n \"run\": {\n \"q_nope\": q_nope,\n \"q_pe\": q_pe,\n \"ckv_cache\": ckv_cache,\n \"kpe_cache\": kpe_cache,\n },\n }\n" } diff --git a/tests/trace/fi_trace_out/mla_paged_decode_h16_ckv512_kpe64_ps64.json b/tests/trace/fi_trace_out/mla_paged_decode_h16_ckv512_kpe64_ps64.json index d4ee7612628..0fa9a6d2c9c 100644 --- a/tests/trace/fi_trace_out/mla_paged_decode_h16_ckv512_kpe64_ps64.json +++ b/tests/trace/fi_trace_out/mla_paged_decode_h16_ckv512_kpe64_ps64.json @@ -28,6 +28,10 @@ "type": "const", "value": 64 }, + "ckv_scale_groups": { + "type": "var", + "description": "Number of 128-channel CKV scale groups." + }, "num_pages": { "type": "var", "description": "Total number of allocated pages in the KV cache." @@ -108,13 +112,23 @@ "shape": null, "dtype": "float32", "optional": true, - "description": "Per-tensor dequantization scale for the compressed-KV cache when kv_data_type is FP8 (real = quantized * ckv_scale). Required together with kpe_scale for the FP8 KV cache path on the fa3 backend. Set during run(), not plan()." + "description": "Per-tensor dequantization scale for the compressed-KV cache when kv_data_type is FP8 (real = quantized * ckv_scale). Required together with kpe_scale for the FP8 KV cache path on the fa2/fa3 backend. Set during run(), not plan()." }, "kpe_scale": { "shape": null, "dtype": "float32", "optional": true, - "description": "Per-tensor dequantization scale for the rope-K cache when kv_data_type is FP8 (real = quantized * kpe_scale). Required together with ckv_scale for the FP8 KV cache path on the fa3 backend. Set during run(), not plan()." + "description": "Per-tensor dequantization scale for the rope-K cache when kv_data_type is FP8 (real = quantized * kpe_scale). Required together with ckv_scale for the FP8 KV cache path on the fa2/fa3 backend. Set during run(), not plan()." + }, + "ckv_scale_arr": { + "shape": [ + "num_pages", + "page_size", + "ckv_scale_groups" + ], + "dtype": "float32", + "optional": true, + "description": "Optional per-token, per-128-channel CKV dequantization scales for the FP8 KV cache path." }, "return_lse": { "shape": null, @@ -144,5 +158,5 @@ }, "reference": "from __future__ import annotations\nimport math\nimport torch\nimport torch.nn.functional as F\n\n@torch.no_grad()\ndef _mla_paged_decode_reference(\n q_nope, q_pe, ckv_cache, kpe_cache, kv_indptr, kv_indices, sm_scale\n):\n batch_size, num_qo_heads, head_dim_ckv = q_nope.shape\n _, _, head_dim_kpe = q_pe.shape\n\n # [num_pages, page_size, head_dim_*] \u2014 keep the page dim; flatten after gather.\n Kc_all = ckv_cache.to(torch.float32)\n Kp_all = kpe_cache.to(torch.float32)\n\n output = torch.zeros(\n (batch_size, num_qo_heads, head_dim_ckv),\n dtype=torch.bfloat16,\n device=q_nope.device,\n )\n lse = torch.full(\n (batch_size, num_qo_heads),\n -float(\"inf\"),\n dtype=torch.float32,\n device=q_nope.device,\n )\n\n for b in range(batch_size):\n page_beg = int(kv_indptr[b].item())\n page_end = int(kv_indptr[b + 1].item())\n if page_beg >= page_end:\n output[b].zero_()\n continue\n # kv_indices are page IDs; gather pages then flatten to a token axis.\n page_ids = kv_indices[page_beg:page_end].to(torch.long)\n Kc = Kc_all[page_ids].reshape(-1, head_dim_ckv) # [L, head_dim_ckv]\n Kp = Kp_all[page_ids].reshape(-1, head_dim_kpe) # [L, head_dim_kpe]\n qn = q_nope[b].to(torch.float32) # [num_qo_heads, head_dim_ckv]\n qp = q_pe[b].to(torch.float32) # [num_qo_heads, head_dim_kpe]\n logits = ((qn @ Kc.T) + (qp @ Kp.T)) * sm_scale # [num_qo_heads, L]\n lse[b] = torch.logsumexp(logits, dim=-1) / math.log(2.0)\n output[b] = (torch.softmax(logits, dim=-1) @ Kc).to(torch.bfloat16)\n\n return output, lse\n", "check": "def standard_check(\n reference_outputs: Any,\n actual_outputs: Any,\n *,\n rtol: Optional[float] = None,\n atol: Optional[float] = None,\n max_mismatch_pct: float = 0.0,\n min_cos_sim: Optional[float] = 1.0 - 1e-3,\n) -> bool:\n \"\"\"Default trace correctness check used when a template does not override it.\"\"\"\n from flashinfer.trace import default_check\n\n return default_check(\n reference_outputs,\n actual_outputs,\n rtol=rtol,\n atol=atol,\n max_mismatch_pct=max_mismatch_pct,\n min_cos_sim=min_cos_sim,\n )\n", - "init": "from __future__ import annotations\nimport math\nimport torch\n\n# ----- shared init helpers -----\n# Copyright (c) 2025 by FlashInfer team.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\n\"\"\"Shared helpers used by ``TraceTemplate.init`` functions.\n\nThis module contains the small set of input-construction patterns that\nrecur across many templates (paged-KV cache index arrays, ragged indptr,\nRoPE pos_ids and cos/sin caches, sampling probs). Each helper is short and\ndocumented; init functions in ``templates/.py`` call into here so\nthe per-template init bodies stay focused on shape/dtype, not boilerplate.\n\nThe full source of this module is **inlined into every dumped JSON's\n``\"init\"`` field** by ``flashinfer/trace/template.py:_render_init_source``,\nso downstream consumers don't need flashinfer installed to re-run the init\nsnippets.\n\"\"\"\n\n\nfrom typing import Optional, Tuple\n\nimport torch\n\n\ndef make_paged_kv_indices(\n batch_size: int,\n num_pages_per_seq: int,\n page_size: int,\n *,\n device: str = \"cuda\",\n) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor]:\n \"\"\"Return ``(kv_indptr, kv_indices, kv_last_page_len)`` for a uniform batch.\n\n Every sequence is assigned exactly ``num_pages_per_seq`` pages, fully\n populated (last-page length == page_size).\n\n Invariants\n ----------\n - ``kv_indptr.shape == (batch_size + 1,)``, dtype int32, monotonic, [0]=0.\n - ``kv_indices == arange(0, batch_size * num_pages_per_seq)``, int32.\n - ``kv_last_page_len == full(batch_size, page_size)``, int32.\n \"\"\"\n total_pages = batch_size * num_pages_per_seq\n kv_indptr = (\n torch.arange(batch_size + 1, dtype=torch.int32, device=device)\n * num_pages_per_seq\n )\n kv_indices = torch.arange(total_pages, dtype=torch.int32, device=device)\n kv_last_page_len = torch.full(\n (batch_size,), page_size, dtype=torch.int32, device=device\n )\n return kv_indptr, kv_indices, kv_last_page_len\n\n\ndef make_ragged_indptr(\n seg_lens,\n *,\n device: str = \"cuda\",\n dtype: torch.dtype = torch.int32,\n) -> torch.Tensor:\n \"\"\"Return cumulative-sum ``indptr`` of length ``len(seg_lens)+1``.\n\n ``seg_lens`` may be a list / tuple / 1-D tensor of segment lengths.\n \"\"\"\n if isinstance(seg_lens, torch.Tensor):\n lens = seg_lens.to(device=device, dtype=dtype)\n else:\n lens = torch.tensor(list(seg_lens), dtype=dtype, device=device)\n indptr = torch.zeros(lens.numel() + 1, dtype=dtype, device=device)\n indptr[1:] = torch.cumsum(lens, dim=0).to(dtype)\n return indptr\n\n\ndef make_uniform_qo_indptr(\n batch_size: int,\n qo_len: int,\n *,\n device: str = \"cuda\",\n) -> torch.Tensor:\n \"\"\"Return ``[0, qo_len, 2*qo_len, ..., batch_size*qo_len]`` int32.\"\"\"\n return torch.arange(batch_size + 1, dtype=torch.int32, device=device) * qo_len\n\n\ndef make_pos_ids(\n nnz: int,\n max_seq_len: Optional[int] = None,\n *,\n device: str = \"cuda\",\n) -> torch.Tensor:\n \"\"\"Return ``[0, 1, ..., nnz-1] (% max_seq_len)`` as int32 on ``device``.\n\n If ``max_seq_len`` is None, no wrapping is applied.\n \"\"\"\n pos = torch.arange(nnz, dtype=torch.int32, device=device)\n if max_seq_len is not None:\n pos = pos % max_seq_len\n return pos\n\n\ndef make_rope_cos_sin_cache(\n max_seq_len: int,\n rope_dim: int,\n *,\n base: float = 1e4,\n device: str = \"cuda\",\n dtype: torch.dtype = torch.float32,\n) -> torch.Tensor:\n \"\"\"Return concatenated ``[cos | sin]`` cache of shape ``[max_seq_len, rope_dim]``.\"\"\"\n t = torch.arange(max_seq_len, dtype=torch.float32, device=device)\n inv = 1.0 / (\n base\n ** (torch.arange(0, rope_dim, 2, dtype=torch.float32, device=device) / rope_dim)\n )\n freqs = t.unsqueeze(-1) * inv.unsqueeze(0)\n cache = torch.cat([torch.cos(freqs), torch.sin(freqs)], dim=-1)\n return cache.to(dtype)\n\n\ndef make_probs(\n batch_size: int,\n vocab_size: int,\n *,\n device: str = \"cuda\",\n dtype: torch.dtype = torch.float32,\n) -> torch.Tensor:\n \"\"\"Return a ``[batch_size, vocab_size]`` probability distribution.\n\n Uses ``softmax(randn(...))`` so each row sums to 1.0. This mirrors the\n pattern used throughout ``tests/utils/test_sampling.py``.\n \"\"\"\n return torch.softmax(\n torch.randn(batch_size, vocab_size, dtype=torch.float32, device=device),\n dim=-1,\n ).to(dtype)\n\n\ndef make_logits(\n batch_size: int,\n vocab_size: int,\n *,\n device: str = \"cuda\",\n dtype: torch.dtype = torch.float32,\n) -> torch.Tensor:\n \"\"\"Return ``randn(batch_size, vocab_size)`` logits.\"\"\"\n return torch.randn(batch_size, vocab_size, dtype=dtype, device=device)\n\n\ndef fp8_safe_randn(\n *shape: int,\n scale: float = 0.1,\n device: str = \"cuda\",\n dtype: torch.dtype = torch.bfloat16,\n) -> torch.Tensor:\n \"\"\"``randn(*shape) * scale`` \u2014 keeps values in the FP8/FP4 representable range.\n\n Tests for fp8/fp4 paths typically multiply ``randn`` by 0.1 to avoid\n saturation when quantizing. Use this helper to mirror that convention.\n \"\"\"\n return (torch.randn(*shape, dtype=dtype, device=device) * scale).to(dtype)\n\n\ndef per_tensor_fp8_quantize(\n x: torch.Tensor,\n *,\n fp8_dtype: torch.dtype = torch.float8_e4m3fn,\n) -> Tuple[torch.Tensor, torch.Tensor]:\n \"\"\"Per-tensor FP8 quantization, mirroring ``tests/utils_fp8.py:to_float8``.\n\n Returns ``(x_fp8, inv_scale)`` where ``inv_scale`` is the dequant\n multiplier (``float \u2248 fp8 * inv_scale``).\n \"\"\"\n finfo = torch.finfo(fp8_dtype)\n amax = x.abs().amax().clamp(min=1e-12)\n scale = finfo.max / amax\n x_q = (x.float() * scale).clamp(min=finfo.min, max=finfo.max).to(fp8_dtype)\n return x_q, scale.float().reciprocal()\n\n\ndef fp8_block_quant_1d(\n x_bf16: torch.Tensor,\n block: int = 128,\n) -> Tuple[torch.Tensor, torch.Tensor]:\n \"\"\"Quantize ``[T, H]`` activations into FP8 with per-``(token, block)``\n column-block scales. Returns ``(x_fp8, scales)`` where\n ``scales`` has shape ``[T, H // block]``.\n\n Mirrors ``_fp8_block_quant_1d`` in\n ``tests/moe/test_dpsk_fused_moe_fp8.py``.\n \"\"\"\n assert x_bf16.dim() == 2\n T, H = x_bf16.shape\n assert H % block == 0\n nb = H // block\n finfo = torch.finfo(torch.float8_e4m3fn)\n max_fp8 = finfo.max\n x_f32 = x_bf16.to(torch.float32)\n x_fp8 = torch.empty((T, H), dtype=torch.float8_e4m3fn, device=x_bf16.device)\n scales = torch.empty((T, nb), dtype=torch.float32, device=x_bf16.device)\n for j in range(nb):\n sl = slice(j * block, (j + 1) * block)\n blk = x_f32[:, sl]\n amax = torch.amax(torch.abs(blk), dim=1)\n s = torch.where(amax > 0, amax / max_fp8, torch.ones_like(amax))\n x_fp8[:, sl] = (blk / s.unsqueeze(1)).to(torch.float8_e4m3fn)\n scales[:, j] = s\n return x_fp8, scales\n\n\ndef fp8_block_quant_2d(\n w_bf16: torch.Tensor,\n block: int = 128,\n) -> Tuple[torch.Tensor, torch.Tensor]:\n \"\"\"Quantize weights ``[..., R, C]`` with 2-D ``block \u00d7 block`` scales.\n\n Returns ``(w_fp8, scales)`` where ``scales`` has shape\n ``[..., R // block, C // block]``. Mirrors ``_fp8_block_quant_2d`` in\n ``tests/moe/test_dpsk_fused_moe_fp8.py``.\n \"\"\"\n assert w_bf16.dim() >= 2\n *prefix, R, C = w_bf16.shape\n assert R % block == 0 and C % block == 0\n nb_r, nb_c = R // block, C // block\n finfo = torch.finfo(torch.float8_e4m3fn)\n max_fp8 = finfo.max\n w_f32 = w_bf16.to(torch.float32).contiguous()\n prefix_ndim = len(prefix)\n reshaped = w_f32.reshape(*prefix, nb_r, block, nb_c, block)\n permute_dims = tuple(range(prefix_ndim)) + (\n prefix_ndim,\n prefix_ndim + 2,\n prefix_ndim + 1,\n prefix_ndim + 3,\n )\n blocks = reshaped.permute(permute_dims).contiguous()\n amax = torch.amax(torch.abs(blocks), dim=(-1, -2))\n scales = torch.where(\n amax > 0, amax / max_fp8, torch.ones_like(amax, dtype=torch.float32)\n )\n q_blocks = (blocks / scales.unsqueeze(-1).unsqueeze(-1)).to(torch.float8_e4m3fn)\n inv_permute = [0] * (prefix_ndim + 4)\n for i, p in enumerate(permute_dims):\n inv_permute[p] = i\n w_fp8 = q_blocks.permute(*inv_permute).reshape(*prefix, R, C).contiguous()\n return w_fp8, scales\n\n\n__all__ = [\n \"make_paged_kv_indices\",\n \"make_ragged_indptr\",\n \"make_uniform_qo_indptr\",\n \"make_pos_ids\",\n \"make_rope_cos_sin_cache\",\n \"make_probs\",\n \"make_logits\",\n \"fp8_safe_randn\",\n \"per_tensor_fp8_quantize\",\n \"fp8_block_quant_1d\",\n \"fp8_block_quant_2d\",\n]\n\n# ----- init -----\ndef _mla_paged_decode_init(\n *,\n batch_size: int,\n num_qo_heads: int = 16,\n head_dim_ckv: int = 512,\n head_dim_kpe: int = 64,\n page_size: int = 64,\n num_pages: int = 0,\n num_pages_per_seq: int = 4,\n len_indptr: int = 0,\n num_kv_indices: int = 0,\n device: str = \"cuda\",\n seed: int = 0,\n):\n \"\"\"Build inputs for ``BatchMLAPagedAttentionWrapper.run()``.\n\n Sourced from ``tests/trace/example.py`` MLA section. Default\n ``num_qo_heads=16`` matches DeepSeek-V3 TP=8.\n \"\"\"\n del num_pages, len_indptr, num_kv_indices\n torch.manual_seed(seed)\n total_pages = batch_size * num_pages_per_seq\n qo_indptr = torch.arange(batch_size + 1, dtype=torch.int32, device=device)\n kv_indptr, kv_indices, _ = make_paged_kv_indices(\n batch_size, num_pages_per_seq, page_size, device=device\n )\n kv_len = torch.full(\n (batch_size,),\n num_pages_per_seq * page_size,\n dtype=torch.int32,\n device=device,\n )\n q_nope = torch.randn(\n batch_size, num_qo_heads, head_dim_ckv, dtype=torch.bfloat16, device=device\n )\n q_pe = torch.randn(\n batch_size, num_qo_heads, head_dim_kpe, dtype=torch.bfloat16, device=device\n )\n ckv_cache = torch.randn(\n total_pages, page_size, head_dim_ckv, dtype=torch.bfloat16, device=device\n )\n kpe_cache = torch.randn(\n total_pages, page_size, head_dim_kpe, dtype=torch.bfloat16, device=device\n )\n return {\n # plan() signature: qo_indptr, kv_indptr, kv_indices, kv_len_arr,\n # num_heads, head_dim_ckv, head_dim_kpe, page_size, causal, ...\n \"plan\": {\n \"qo_indptr\": qo_indptr,\n \"kv_indptr\": kv_indptr,\n \"kv_indices\": kv_indices,\n \"kv_len_arr\": kv_len,\n \"num_heads\": int(num_qo_heads),\n \"head_dim_ckv\": int(head_dim_ckv),\n \"head_dim_kpe\": int(head_dim_kpe),\n \"page_size\": int(page_size),\n \"causal\": False,\n # tests/attention/test_deepseek_mla.py scales by the pre-absorption\n # QK dimension: qk_nope_head_dim(128) + head_dim_kpe.\n \"sm_scale\": 1.0 / ((128 + head_dim_kpe) ** 0.5),\n \"q_data_type\": torch.bfloat16,\n \"kv_data_type\": torch.bfloat16,\n },\n \"run\": {\n \"q_nope\": q_nope,\n \"q_pe\": q_pe,\n \"ckv_cache\": ckv_cache,\n \"kpe_cache\": kpe_cache,\n },\n }\n" + "init": "from __future__ import annotations\nimport math\nimport torch\n\n# ----- shared init helpers -----\n# Copyright (c) 2025 by FlashInfer team.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\n\"\"\"Shared helpers used by ``TraceTemplate.init`` functions.\n\nThis module contains the small set of input-construction patterns that\nrecur across many templates (paged-KV cache index arrays, ragged indptr,\nRoPE pos_ids and cos/sin caches, sampling probs). Each helper is short and\ndocumented; init functions in ``templates/.py`` call into here so\nthe per-template init bodies stay focused on shape/dtype, not boilerplate.\n\nThe full source of this module is **inlined into every dumped JSON's\n``\"init\"`` field** by ``flashinfer/trace/template.py:_render_init_source``,\nso downstream consumers don't need flashinfer installed to re-run the init\nsnippets.\n\"\"\"\n\n\nfrom typing import Optional, Tuple\n\nimport torch\n\n\ndef make_paged_kv_indices(\n batch_size: int,\n num_pages_per_seq: int,\n page_size: int,\n *,\n device: str = \"cuda\",\n) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor]:\n \"\"\"Return ``(kv_indptr, kv_indices, kv_last_page_len)`` for a uniform batch.\n\n Every sequence is assigned exactly ``num_pages_per_seq`` pages, fully\n populated (last-page length == page_size).\n\n Invariants\n ----------\n - ``kv_indptr.shape == (batch_size + 1,)``, dtype int32, monotonic, [0]=0.\n - ``kv_indices == arange(0, batch_size * num_pages_per_seq)``, int32.\n - ``kv_last_page_len == full(batch_size, page_size)``, int32.\n \"\"\"\n total_pages = batch_size * num_pages_per_seq\n kv_indptr = (\n torch.arange(batch_size + 1, dtype=torch.int32, device=device)\n * num_pages_per_seq\n )\n kv_indices = torch.arange(total_pages, dtype=torch.int32, device=device)\n kv_last_page_len = torch.full(\n (batch_size,), page_size, dtype=torch.int32, device=device\n )\n return kv_indptr, kv_indices, kv_last_page_len\n\n\ndef make_ragged_indptr(\n seg_lens,\n *,\n device: str = \"cuda\",\n dtype: torch.dtype = torch.int32,\n) -> torch.Tensor:\n \"\"\"Return cumulative-sum ``indptr`` of length ``len(seg_lens)+1``.\n\n ``seg_lens`` may be a list / tuple / 1-D tensor of segment lengths.\n \"\"\"\n if isinstance(seg_lens, torch.Tensor):\n lens = seg_lens.to(device=device, dtype=dtype)\n else:\n lens = torch.tensor(list(seg_lens), dtype=dtype, device=device)\n indptr = torch.zeros(lens.numel() + 1, dtype=dtype, device=device)\n indptr[1:] = torch.cumsum(lens, dim=0).to(dtype)\n return indptr\n\n\ndef make_uniform_qo_indptr(\n batch_size: int,\n qo_len: int,\n *,\n device: str = \"cuda\",\n) -> torch.Tensor:\n \"\"\"Return ``[0, qo_len, 2*qo_len, ..., batch_size*qo_len]`` int32.\"\"\"\n return torch.arange(batch_size + 1, dtype=torch.int32, device=device) * qo_len\n\n\ndef make_pos_ids(\n nnz: int,\n max_seq_len: Optional[int] = None,\n *,\n device: str = \"cuda\",\n) -> torch.Tensor:\n \"\"\"Return ``[0, 1, ..., nnz-1] (% max_seq_len)`` as int32 on ``device``.\n\n If ``max_seq_len`` is None, no wrapping is applied.\n \"\"\"\n pos = torch.arange(nnz, dtype=torch.int32, device=device)\n if max_seq_len is not None:\n pos = pos % max_seq_len\n return pos\n\n\ndef make_rope_cos_sin_cache(\n max_seq_len: int,\n rope_dim: int,\n *,\n base: float = 1e4,\n device: str = \"cuda\",\n dtype: torch.dtype = torch.float32,\n) -> torch.Tensor:\n \"\"\"Return concatenated ``[cos | sin]`` cache of shape ``[max_seq_len, rope_dim]``.\"\"\"\n t = torch.arange(max_seq_len, dtype=torch.float32, device=device)\n inv = 1.0 / (\n base\n ** (torch.arange(0, rope_dim, 2, dtype=torch.float32, device=device) / rope_dim)\n )\n freqs = t.unsqueeze(-1) * inv.unsqueeze(0)\n cache = torch.cat([torch.cos(freqs), torch.sin(freqs)], dim=-1)\n return cache.to(dtype)\n\n\ndef make_probs(\n batch_size: int,\n vocab_size: int,\n *,\n device: str = \"cuda\",\n dtype: torch.dtype = torch.float32,\n) -> torch.Tensor:\n \"\"\"Return a ``[batch_size, vocab_size]`` probability distribution.\n\n Uses ``softmax(randn(...))`` so each row sums to 1.0. This mirrors the\n pattern used throughout ``tests/utils/test_sampling.py``.\n \"\"\"\n return torch.softmax(\n torch.randn(batch_size, vocab_size, dtype=torch.float32, device=device),\n dim=-1,\n ).to(dtype)\n\n\ndef make_logits(\n batch_size: int,\n vocab_size: int,\n *,\n device: str = \"cuda\",\n dtype: torch.dtype = torch.float32,\n) -> torch.Tensor:\n \"\"\"Return ``randn(batch_size, vocab_size)`` logits.\"\"\"\n return torch.randn(batch_size, vocab_size, dtype=dtype, device=device)\n\n\ndef fp8_safe_randn(\n *shape: int,\n scale: float = 0.1,\n device: str = \"cuda\",\n dtype: torch.dtype = torch.bfloat16,\n) -> torch.Tensor:\n \"\"\"``randn(*shape) * scale`` \u2014 keeps values in the FP8/FP4 representable range.\n\n Tests for fp8/fp4 paths typically multiply ``randn`` by 0.1 to avoid\n saturation when quantizing. Use this helper to mirror that convention.\n \"\"\"\n return (torch.randn(*shape, dtype=dtype, device=device) * scale).to(dtype)\n\n\ndef per_tensor_fp8_quantize(\n x: torch.Tensor,\n *,\n fp8_dtype: torch.dtype = torch.float8_e4m3fn,\n) -> Tuple[torch.Tensor, torch.Tensor]:\n \"\"\"Per-tensor FP8 quantization, mirroring ``tests/utils_fp8.py:to_float8``.\n\n Returns ``(x_fp8, inv_scale)`` where ``inv_scale`` is the dequant\n multiplier (``float \u2248 fp8 * inv_scale``).\n \"\"\"\n finfo = torch.finfo(fp8_dtype)\n amax = x.abs().amax().clamp(min=1e-12)\n scale = finfo.max / amax\n x_q = (x.float() * scale).clamp(min=finfo.min, max=finfo.max).to(fp8_dtype)\n return x_q, scale.float().reciprocal()\n\n\ndef fp8_block_quant_1d(\n x_bf16: torch.Tensor,\n block: int = 128,\n) -> Tuple[torch.Tensor, torch.Tensor]:\n \"\"\"Quantize ``[T, H]`` activations into FP8 with per-``(token, block)``\n column-block scales. Returns ``(x_fp8, scales)`` where\n ``scales`` has shape ``[T, H // block]``.\n\n Mirrors ``_fp8_block_quant_1d`` in\n ``tests/moe/test_dpsk_fused_moe_fp8.py``.\n \"\"\"\n assert x_bf16.dim() == 2\n T, H = x_bf16.shape\n assert H % block == 0\n nb = H // block\n finfo = torch.finfo(torch.float8_e4m3fn)\n max_fp8 = finfo.max\n x_f32 = x_bf16.to(torch.float32)\n x_fp8 = torch.empty((T, H), dtype=torch.float8_e4m3fn, device=x_bf16.device)\n scales = torch.empty((T, nb), dtype=torch.float32, device=x_bf16.device)\n for j in range(nb):\n sl = slice(j * block, (j + 1) * block)\n blk = x_f32[:, sl]\n amax = torch.amax(torch.abs(blk), dim=1)\n s = torch.where(amax > 0, amax / max_fp8, torch.ones_like(amax))\n x_fp8[:, sl] = (blk / s.unsqueeze(1)).to(torch.float8_e4m3fn)\n scales[:, j] = s\n return x_fp8, scales\n\n\ndef fp8_block_quant_2d(\n w_bf16: torch.Tensor,\n block: int = 128,\n) -> Tuple[torch.Tensor, torch.Tensor]:\n \"\"\"Quantize weights ``[..., R, C]`` with 2-D ``block \u00d7 block`` scales.\n\n Returns ``(w_fp8, scales)`` where ``scales`` has shape\n ``[..., R // block, C // block]``. Mirrors ``_fp8_block_quant_2d`` in\n ``tests/moe/test_dpsk_fused_moe_fp8.py``.\n \"\"\"\n assert w_bf16.dim() >= 2\n *prefix, R, C = w_bf16.shape\n assert R % block == 0 and C % block == 0\n nb_r, nb_c = R // block, C // block\n finfo = torch.finfo(torch.float8_e4m3fn)\n max_fp8 = finfo.max\n w_f32 = w_bf16.to(torch.float32).contiguous()\n prefix_ndim = len(prefix)\n reshaped = w_f32.reshape(*prefix, nb_r, block, nb_c, block)\n permute_dims = tuple(range(prefix_ndim)) + (\n prefix_ndim,\n prefix_ndim + 2,\n prefix_ndim + 1,\n prefix_ndim + 3,\n )\n blocks = reshaped.permute(permute_dims).contiguous()\n amax = torch.amax(torch.abs(blocks), dim=(-1, -2))\n scales = torch.where(\n amax > 0, amax / max_fp8, torch.ones_like(amax, dtype=torch.float32)\n )\n q_blocks = (blocks / scales.unsqueeze(-1).unsqueeze(-1)).to(torch.float8_e4m3fn)\n inv_permute = [0] * (prefix_ndim + 4)\n for i, p in enumerate(permute_dims):\n inv_permute[p] = i\n w_fp8 = q_blocks.permute(*inv_permute).reshape(*prefix, R, C).contiguous()\n return w_fp8, scales\n\n\n__all__ = [\n \"make_paged_kv_indices\",\n \"make_ragged_indptr\",\n \"make_uniform_qo_indptr\",\n \"make_pos_ids\",\n \"make_rope_cos_sin_cache\",\n \"make_probs\",\n \"make_logits\",\n \"fp8_safe_randn\",\n \"per_tensor_fp8_quantize\",\n \"fp8_block_quant_1d\",\n \"fp8_block_quant_2d\",\n]\n\n# ----- init -----\ndef _mla_paged_decode_init(\n *,\n batch_size: int,\n num_qo_heads: int = 16,\n head_dim_ckv: int = 512,\n head_dim_kpe: int = 64,\n page_size: int = 64,\n num_pages: int = 0,\n num_pages_per_seq: int = 4,\n len_indptr: int = 0,\n num_kv_indices: int = 0,\n ckv_scale_groups: int = 0,\n device: str = \"cuda\",\n seed: int = 0,\n):\n \"\"\"Build inputs for ``BatchMLAPagedAttentionWrapper.run()``.\n\n Sourced from ``tests/trace/example.py`` MLA section. Default\n ``num_qo_heads=16`` matches DeepSeek-V3 TP=8.\n \"\"\"\n del num_pages, len_indptr, num_kv_indices, ckv_scale_groups\n torch.manual_seed(seed)\n total_pages = batch_size * num_pages_per_seq\n qo_indptr = torch.arange(batch_size + 1, dtype=torch.int32, device=device)\n kv_indptr, kv_indices, _ = make_paged_kv_indices(\n batch_size, num_pages_per_seq, page_size, device=device\n )\n kv_len = torch.full(\n (batch_size,),\n num_pages_per_seq * page_size,\n dtype=torch.int32,\n device=device,\n )\n q_nope = torch.randn(\n batch_size, num_qo_heads, head_dim_ckv, dtype=torch.bfloat16, device=device\n )\n q_pe = torch.randn(\n batch_size, num_qo_heads, head_dim_kpe, dtype=torch.bfloat16, device=device\n )\n ckv_cache = torch.randn(\n total_pages, page_size, head_dim_ckv, dtype=torch.bfloat16, device=device\n )\n kpe_cache = torch.randn(\n total_pages, page_size, head_dim_kpe, dtype=torch.bfloat16, device=device\n )\n return {\n # plan() signature: qo_indptr, kv_indptr, kv_indices, kv_len_arr,\n # num_heads, head_dim_ckv, head_dim_kpe, page_size, causal, ...\n \"plan\": {\n \"qo_indptr\": qo_indptr,\n \"kv_indptr\": kv_indptr,\n \"kv_indices\": kv_indices,\n \"kv_len_arr\": kv_len,\n \"num_heads\": int(num_qo_heads),\n \"head_dim_ckv\": int(head_dim_ckv),\n \"head_dim_kpe\": int(head_dim_kpe),\n \"page_size\": int(page_size),\n \"causal\": False,\n # tests/attention/test_deepseek_mla.py scales by the pre-absorption\n # QK dimension: qk_nope_head_dim(128) + head_dim_kpe.\n \"sm_scale\": 1.0 / ((128 + head_dim_kpe) ** 0.5),\n \"q_data_type\": torch.bfloat16,\n \"kv_data_type\": torch.bfloat16,\n },\n \"run\": {\n \"q_nope\": q_nope,\n \"q_pe\": q_pe,\n \"ckv_cache\": ckv_cache,\n \"kpe_cache\": kpe_cache,\n },\n }\n" } diff --git a/tests/trace/test_fi_trace.py b/tests/trace/test_fi_trace.py index 26afdbc4697..126e77af978 100644 --- a/tests/trace/test_fi_trace.py +++ b/tests/trace/test_fi_trace.py @@ -680,9 +680,14 @@ def test_mla_paged_fi_trace(): q_pe = torch.randn(batch_size, num_qo_heads, head_dim_kpe, dtype=torch.bfloat16) ckv_cache = torch.randn(num_pages, page_size, head_dim_ckv, dtype=torch.bfloat16) kpe_cache = torch.randn(num_pages, page_size, head_dim_kpe, dtype=torch.bfloat16) + ckv_scale_arr = torch.randn(num_pages, page_size, head_dim_ckv // 128) defn = BatchMLAPagedAttentionWrapper.run.fi_trace( - q_nope=q_nope, q_pe=q_pe, ckv_cache=ckv_cache, kpe_cache=kpe_cache + q_nope=q_nope, + q_pe=q_pe, + ckv_cache=ckv_cache, + kpe_cache=kpe_cache, + ckv_scale_arr=ckv_scale_arr, ) _check_defn(defn, "mla_paged", "BatchMLAPagedAttentionWrapper") axes = defn["axes"] @@ -690,6 +695,12 @@ def test_mla_paged_fi_trace(): assert axes["head_dim_ckv"]["value"] == head_dim_ckv assert axes["head_dim_kpe"]["value"] == head_dim_kpe assert axes["page_size"]["value"] == page_size + assert axes["ckv_scale_groups"]["type"] == "var" + assert defn["inputs"]["ckv_scale_arr"]["shape"] == [ + "num_pages", + "page_size", + "ckv_scale_groups", + ] # --------------------------------------------------------------------------- From b58678992e9f2d7151984d885a5b4785f7353858 Mon Sep 17 00:00:00 2001 From: Xinyuan Tong Date: Thu, 13 Aug 2026 06:15:49 +0000 Subject: [PATCH 4/5] fix: support FP8 scales in MLA trace reference --- flashinfer/trace/templates/attention.py | 24 +++++++- ...mla_paged_decode_h16_ckv512_kpe64_ps1.json | 2 +- ...la_paged_decode_h16_ckv512_kpe64_ps64.json | 2 +- tests/trace/test_fi_trace.py | 19 +++++-- ..._mla_paged_decode_reference_correctness.py | 56 +++++++++++++++++++ 5 files changed, 96 insertions(+), 7 deletions(-) create mode 100644 tests/trace/test_mla_paged_decode_reference_correctness.py diff --git a/flashinfer/trace/templates/attention.py b/flashinfer/trace/templates/attention.py index ab8799efff8..6b088865d29 100644 --- a/flashinfer/trace/templates/attention.py +++ b/flashinfer/trace/templates/attention.py @@ -1374,14 +1374,36 @@ def _split_into_indptr(total: int) -> torch.Tensor: @torch.no_grad() def _mla_paged_decode_reference( - q_nope, q_pe, ckv_cache, kpe_cache, kv_indptr, kv_indices, sm_scale + q_nope, + q_pe, + ckv_cache, + kpe_cache, + kv_indptr, + kv_indices, + sm_scale, + ckv_scale=None, + kpe_scale=None, + ckv_scale_arr=None, + return_lse=False, ): + del return_lse batch_size, num_qo_heads, head_dim_ckv = q_nope.shape _, _, head_dim_kpe = q_pe.shape # [num_pages, page_size, head_dim_*] — keep the page dim; flatten after gather. Kc_all = ckv_cache.to(torch.float32) Kp_all = kpe_cache.to(torch.float32) + if ckv_cache.dtype == torch.float8_e4m3fn: + if ckv_scale is None or kpe_scale is None: + raise ValueError("ckv_scale and kpe_scale are required for FP8 KV cache") + if ckv_scale_arr is not None: + Kc_all = ( + Kc_all.reshape(*Kc_all.shape[:-1], head_dim_ckv // 128, 128) + * ckv_scale_arr.unsqueeze(-1) + ).reshape_as(Kc_all) + else: + Kc_all *= float(ckv_scale) + Kp_all *= float(kpe_scale) output = torch.zeros( (batch_size, num_qo_heads, head_dim_ckv), diff --git a/tests/trace/fi_trace_out/mla_paged_decode_h16_ckv512_kpe64_ps1.json b/tests/trace/fi_trace_out/mla_paged_decode_h16_ckv512_kpe64_ps1.json index a6f3ad7ec55..e64b6f38e14 100644 --- a/tests/trace/fi_trace_out/mla_paged_decode_h16_ckv512_kpe64_ps1.json +++ b/tests/trace/fi_trace_out/mla_paged_decode_h16_ckv512_kpe64_ps1.json @@ -156,7 +156,7 @@ "description": "The 2-based log-sum-exp of attention logits." } }, - "reference": "from __future__ import annotations\nimport math\nimport torch\nimport torch.nn.functional as F\n\n@torch.no_grad()\ndef _mla_paged_decode_reference(\n q_nope, q_pe, ckv_cache, kpe_cache, kv_indptr, kv_indices, sm_scale\n):\n batch_size, num_qo_heads, head_dim_ckv = q_nope.shape\n _, _, head_dim_kpe = q_pe.shape\n\n # [num_pages, page_size, head_dim_*] \u2014 keep the page dim; flatten after gather.\n Kc_all = ckv_cache.to(torch.float32)\n Kp_all = kpe_cache.to(torch.float32)\n\n output = torch.zeros(\n (batch_size, num_qo_heads, head_dim_ckv),\n dtype=torch.bfloat16,\n device=q_nope.device,\n )\n lse = torch.full(\n (batch_size, num_qo_heads),\n -float(\"inf\"),\n dtype=torch.float32,\n device=q_nope.device,\n )\n\n for b in range(batch_size):\n page_beg = int(kv_indptr[b].item())\n page_end = int(kv_indptr[b + 1].item())\n if page_beg >= page_end:\n output[b].zero_()\n continue\n # kv_indices are page IDs; gather pages then flatten to a token axis.\n page_ids = kv_indices[page_beg:page_end].to(torch.long)\n Kc = Kc_all[page_ids].reshape(-1, head_dim_ckv) # [L, head_dim_ckv]\n Kp = Kp_all[page_ids].reshape(-1, head_dim_kpe) # [L, head_dim_kpe]\n qn = q_nope[b].to(torch.float32) # [num_qo_heads, head_dim_ckv]\n qp = q_pe[b].to(torch.float32) # [num_qo_heads, head_dim_kpe]\n logits = ((qn @ Kc.T) + (qp @ Kp.T)) * sm_scale # [num_qo_heads, L]\n lse[b] = torch.logsumexp(logits, dim=-1) / math.log(2.0)\n output[b] = (torch.softmax(logits, dim=-1) @ Kc).to(torch.bfloat16)\n\n return output, lse\n", + "reference": "from __future__ import annotations\nimport math\nimport torch\nimport torch.nn.functional as F\n\n@torch.no_grad()\ndef _mla_paged_decode_reference(\n q_nope,\n q_pe,\n ckv_cache,\n kpe_cache,\n kv_indptr,\n kv_indices,\n sm_scale,\n ckv_scale=None,\n kpe_scale=None,\n ckv_scale_arr=None,\n return_lse=False,\n):\n del return_lse\n batch_size, num_qo_heads, head_dim_ckv = q_nope.shape\n _, _, head_dim_kpe = q_pe.shape\n\n # [num_pages, page_size, head_dim_*] \u2014 keep the page dim; flatten after gather.\n Kc_all = ckv_cache.to(torch.float32)\n Kp_all = kpe_cache.to(torch.float32)\n if ckv_cache.dtype == torch.float8_e4m3fn:\n if ckv_scale is None or kpe_scale is None:\n raise ValueError(\"ckv_scale and kpe_scale are required for FP8 KV cache\")\n if ckv_scale_arr is not None:\n Kc_all = (\n Kc_all.reshape(*Kc_all.shape[:-1], head_dim_ckv // 128, 128)\n * ckv_scale_arr.unsqueeze(-1)\n ).reshape_as(Kc_all)\n else:\n Kc_all *= float(ckv_scale)\n Kp_all *= float(kpe_scale)\n\n output = torch.zeros(\n (batch_size, num_qo_heads, head_dim_ckv),\n dtype=torch.bfloat16,\n device=q_nope.device,\n )\n lse = torch.full(\n (batch_size, num_qo_heads),\n -float(\"inf\"),\n dtype=torch.float32,\n device=q_nope.device,\n )\n\n for b in range(batch_size):\n page_beg = int(kv_indptr[b].item())\n page_end = int(kv_indptr[b + 1].item())\n if page_beg >= page_end:\n output[b].zero_()\n continue\n # kv_indices are page IDs; gather pages then flatten to a token axis.\n page_ids = kv_indices[page_beg:page_end].to(torch.long)\n Kc = Kc_all[page_ids].reshape(-1, head_dim_ckv) # [L, head_dim_ckv]\n Kp = Kp_all[page_ids].reshape(-1, head_dim_kpe) # [L, head_dim_kpe]\n qn = q_nope[b].to(torch.float32) # [num_qo_heads, head_dim_ckv]\n qp = q_pe[b].to(torch.float32) # [num_qo_heads, head_dim_kpe]\n logits = ((qn @ Kc.T) + (qp @ Kp.T)) * sm_scale # [num_qo_heads, L]\n lse[b] = torch.logsumexp(logits, dim=-1) / math.log(2.0)\n output[b] = (torch.softmax(logits, dim=-1) @ Kc).to(torch.bfloat16)\n\n return output, lse\n", "check": "def standard_check(\n reference_outputs: Any,\n actual_outputs: Any,\n *,\n rtol: Optional[float] = None,\n atol: Optional[float] = None,\n max_mismatch_pct: float = 0.0,\n min_cos_sim: Optional[float] = 1.0 - 1e-3,\n) -> bool:\n \"\"\"Default trace correctness check used when a template does not override it.\"\"\"\n from flashinfer.trace import default_check\n\n return default_check(\n reference_outputs,\n actual_outputs,\n rtol=rtol,\n atol=atol,\n max_mismatch_pct=max_mismatch_pct,\n min_cos_sim=min_cos_sim,\n )\n", "init": "from __future__ import annotations\nimport math\nimport torch\n\n# ----- shared init helpers -----\n# Copyright (c) 2025 by FlashInfer team.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\n\"\"\"Shared helpers used by ``TraceTemplate.init`` functions.\n\nThis module contains the small set of input-construction patterns that\nrecur across many templates (paged-KV cache index arrays, ragged indptr,\nRoPE pos_ids and cos/sin caches, sampling probs). Each helper is short and\ndocumented; init functions in ``templates/.py`` call into here so\nthe per-template init bodies stay focused on shape/dtype, not boilerplate.\n\nThe full source of this module is **inlined into every dumped JSON's\n``\"init\"`` field** by ``flashinfer/trace/template.py:_render_init_source``,\nso downstream consumers don't need flashinfer installed to re-run the init\nsnippets.\n\"\"\"\n\n\nfrom typing import Optional, Tuple\n\nimport torch\n\n\ndef make_paged_kv_indices(\n batch_size: int,\n num_pages_per_seq: int,\n page_size: int,\n *,\n device: str = \"cuda\",\n) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor]:\n \"\"\"Return ``(kv_indptr, kv_indices, kv_last_page_len)`` for a uniform batch.\n\n Every sequence is assigned exactly ``num_pages_per_seq`` pages, fully\n populated (last-page length == page_size).\n\n Invariants\n ----------\n - ``kv_indptr.shape == (batch_size + 1,)``, dtype int32, monotonic, [0]=0.\n - ``kv_indices == arange(0, batch_size * num_pages_per_seq)``, int32.\n - ``kv_last_page_len == full(batch_size, page_size)``, int32.\n \"\"\"\n total_pages = batch_size * num_pages_per_seq\n kv_indptr = (\n torch.arange(batch_size + 1, dtype=torch.int32, device=device)\n * num_pages_per_seq\n )\n kv_indices = torch.arange(total_pages, dtype=torch.int32, device=device)\n kv_last_page_len = torch.full(\n (batch_size,), page_size, dtype=torch.int32, device=device\n )\n return kv_indptr, kv_indices, kv_last_page_len\n\n\ndef make_ragged_indptr(\n seg_lens,\n *,\n device: str = \"cuda\",\n dtype: torch.dtype = torch.int32,\n) -> torch.Tensor:\n \"\"\"Return cumulative-sum ``indptr`` of length ``len(seg_lens)+1``.\n\n ``seg_lens`` may be a list / tuple / 1-D tensor of segment lengths.\n \"\"\"\n if isinstance(seg_lens, torch.Tensor):\n lens = seg_lens.to(device=device, dtype=dtype)\n else:\n lens = torch.tensor(list(seg_lens), dtype=dtype, device=device)\n indptr = torch.zeros(lens.numel() + 1, dtype=dtype, device=device)\n indptr[1:] = torch.cumsum(lens, dim=0).to(dtype)\n return indptr\n\n\ndef make_uniform_qo_indptr(\n batch_size: int,\n qo_len: int,\n *,\n device: str = \"cuda\",\n) -> torch.Tensor:\n \"\"\"Return ``[0, qo_len, 2*qo_len, ..., batch_size*qo_len]`` int32.\"\"\"\n return torch.arange(batch_size + 1, dtype=torch.int32, device=device) * qo_len\n\n\ndef make_pos_ids(\n nnz: int,\n max_seq_len: Optional[int] = None,\n *,\n device: str = \"cuda\",\n) -> torch.Tensor:\n \"\"\"Return ``[0, 1, ..., nnz-1] (% max_seq_len)`` as int32 on ``device``.\n\n If ``max_seq_len`` is None, no wrapping is applied.\n \"\"\"\n pos = torch.arange(nnz, dtype=torch.int32, device=device)\n if max_seq_len is not None:\n pos = pos % max_seq_len\n return pos\n\n\ndef make_rope_cos_sin_cache(\n max_seq_len: int,\n rope_dim: int,\n *,\n base: float = 1e4,\n device: str = \"cuda\",\n dtype: torch.dtype = torch.float32,\n) -> torch.Tensor:\n \"\"\"Return concatenated ``[cos | sin]`` cache of shape ``[max_seq_len, rope_dim]``.\"\"\"\n t = torch.arange(max_seq_len, dtype=torch.float32, device=device)\n inv = 1.0 / (\n base\n ** (torch.arange(0, rope_dim, 2, dtype=torch.float32, device=device) / rope_dim)\n )\n freqs = t.unsqueeze(-1) * inv.unsqueeze(0)\n cache = torch.cat([torch.cos(freqs), torch.sin(freqs)], dim=-1)\n return cache.to(dtype)\n\n\ndef make_probs(\n batch_size: int,\n vocab_size: int,\n *,\n device: str = \"cuda\",\n dtype: torch.dtype = torch.float32,\n) -> torch.Tensor:\n \"\"\"Return a ``[batch_size, vocab_size]`` probability distribution.\n\n Uses ``softmax(randn(...))`` so each row sums to 1.0. This mirrors the\n pattern used throughout ``tests/utils/test_sampling.py``.\n \"\"\"\n return torch.softmax(\n torch.randn(batch_size, vocab_size, dtype=torch.float32, device=device),\n dim=-1,\n ).to(dtype)\n\n\ndef make_logits(\n batch_size: int,\n vocab_size: int,\n *,\n device: str = \"cuda\",\n dtype: torch.dtype = torch.float32,\n) -> torch.Tensor:\n \"\"\"Return ``randn(batch_size, vocab_size)`` logits.\"\"\"\n return torch.randn(batch_size, vocab_size, dtype=dtype, device=device)\n\n\ndef fp8_safe_randn(\n *shape: int,\n scale: float = 0.1,\n device: str = \"cuda\",\n dtype: torch.dtype = torch.bfloat16,\n) -> torch.Tensor:\n \"\"\"``randn(*shape) * scale`` \u2014 keeps values in the FP8/FP4 representable range.\n\n Tests for fp8/fp4 paths typically multiply ``randn`` by 0.1 to avoid\n saturation when quantizing. Use this helper to mirror that convention.\n \"\"\"\n return (torch.randn(*shape, dtype=dtype, device=device) * scale).to(dtype)\n\n\ndef per_tensor_fp8_quantize(\n x: torch.Tensor,\n *,\n fp8_dtype: torch.dtype = torch.float8_e4m3fn,\n) -> Tuple[torch.Tensor, torch.Tensor]:\n \"\"\"Per-tensor FP8 quantization, mirroring ``tests/utils_fp8.py:to_float8``.\n\n Returns ``(x_fp8, inv_scale)`` where ``inv_scale`` is the dequant\n multiplier (``float \u2248 fp8 * inv_scale``).\n \"\"\"\n finfo = torch.finfo(fp8_dtype)\n amax = x.abs().amax().clamp(min=1e-12)\n scale = finfo.max / amax\n x_q = (x.float() * scale).clamp(min=finfo.min, max=finfo.max).to(fp8_dtype)\n return x_q, scale.float().reciprocal()\n\n\ndef fp8_block_quant_1d(\n x_bf16: torch.Tensor,\n block: int = 128,\n) -> Tuple[torch.Tensor, torch.Tensor]:\n \"\"\"Quantize ``[T, H]`` activations into FP8 with per-``(token, block)``\n column-block scales. Returns ``(x_fp8, scales)`` where\n ``scales`` has shape ``[T, H // block]``.\n\n Mirrors ``_fp8_block_quant_1d`` in\n ``tests/moe/test_dpsk_fused_moe_fp8.py``.\n \"\"\"\n assert x_bf16.dim() == 2\n T, H = x_bf16.shape\n assert H % block == 0\n nb = H // block\n finfo = torch.finfo(torch.float8_e4m3fn)\n max_fp8 = finfo.max\n x_f32 = x_bf16.to(torch.float32)\n x_fp8 = torch.empty((T, H), dtype=torch.float8_e4m3fn, device=x_bf16.device)\n scales = torch.empty((T, nb), dtype=torch.float32, device=x_bf16.device)\n for j in range(nb):\n sl = slice(j * block, (j + 1) * block)\n blk = x_f32[:, sl]\n amax = torch.amax(torch.abs(blk), dim=1)\n s = torch.where(amax > 0, amax / max_fp8, torch.ones_like(amax))\n x_fp8[:, sl] = (blk / s.unsqueeze(1)).to(torch.float8_e4m3fn)\n scales[:, j] = s\n return x_fp8, scales\n\n\ndef fp8_block_quant_2d(\n w_bf16: torch.Tensor,\n block: int = 128,\n) -> Tuple[torch.Tensor, torch.Tensor]:\n \"\"\"Quantize weights ``[..., R, C]`` with 2-D ``block \u00d7 block`` scales.\n\n Returns ``(w_fp8, scales)`` where ``scales`` has shape\n ``[..., R // block, C // block]``. Mirrors ``_fp8_block_quant_2d`` in\n ``tests/moe/test_dpsk_fused_moe_fp8.py``.\n \"\"\"\n assert w_bf16.dim() >= 2\n *prefix, R, C = w_bf16.shape\n assert R % block == 0 and C % block == 0\n nb_r, nb_c = R // block, C // block\n finfo = torch.finfo(torch.float8_e4m3fn)\n max_fp8 = finfo.max\n w_f32 = w_bf16.to(torch.float32).contiguous()\n prefix_ndim = len(prefix)\n reshaped = w_f32.reshape(*prefix, nb_r, block, nb_c, block)\n permute_dims = tuple(range(prefix_ndim)) + (\n prefix_ndim,\n prefix_ndim + 2,\n prefix_ndim + 1,\n prefix_ndim + 3,\n )\n blocks = reshaped.permute(permute_dims).contiguous()\n amax = torch.amax(torch.abs(blocks), dim=(-1, -2))\n scales = torch.where(\n amax > 0, amax / max_fp8, torch.ones_like(amax, dtype=torch.float32)\n )\n q_blocks = (blocks / scales.unsqueeze(-1).unsqueeze(-1)).to(torch.float8_e4m3fn)\n inv_permute = [0] * (prefix_ndim + 4)\n for i, p in enumerate(permute_dims):\n inv_permute[p] = i\n w_fp8 = q_blocks.permute(*inv_permute).reshape(*prefix, R, C).contiguous()\n return w_fp8, scales\n\n\n__all__ = [\n \"make_paged_kv_indices\",\n \"make_ragged_indptr\",\n \"make_uniform_qo_indptr\",\n \"make_pos_ids\",\n \"make_rope_cos_sin_cache\",\n \"make_probs\",\n \"make_logits\",\n \"fp8_safe_randn\",\n \"per_tensor_fp8_quantize\",\n \"fp8_block_quant_1d\",\n \"fp8_block_quant_2d\",\n]\n\n# ----- init -----\ndef _mla_paged_decode_init(\n *,\n batch_size: int,\n num_qo_heads: int = 16,\n head_dim_ckv: int = 512,\n head_dim_kpe: int = 64,\n page_size: int = 64,\n num_pages: int = 0,\n num_pages_per_seq: int = 4,\n len_indptr: int = 0,\n num_kv_indices: int = 0,\n ckv_scale_groups: int = 0,\n device: str = \"cuda\",\n seed: int = 0,\n):\n \"\"\"Build inputs for ``BatchMLAPagedAttentionWrapper.run()``.\n\n Sourced from ``tests/trace/example.py`` MLA section. Default\n ``num_qo_heads=16`` matches DeepSeek-V3 TP=8.\n \"\"\"\n del num_pages, len_indptr, num_kv_indices, ckv_scale_groups\n torch.manual_seed(seed)\n total_pages = batch_size * num_pages_per_seq\n qo_indptr = torch.arange(batch_size + 1, dtype=torch.int32, device=device)\n kv_indptr, kv_indices, _ = make_paged_kv_indices(\n batch_size, num_pages_per_seq, page_size, device=device\n )\n kv_len = torch.full(\n (batch_size,),\n num_pages_per_seq * page_size,\n dtype=torch.int32,\n device=device,\n )\n q_nope = torch.randn(\n batch_size, num_qo_heads, head_dim_ckv, dtype=torch.bfloat16, device=device\n )\n q_pe = torch.randn(\n batch_size, num_qo_heads, head_dim_kpe, dtype=torch.bfloat16, device=device\n )\n ckv_cache = torch.randn(\n total_pages, page_size, head_dim_ckv, dtype=torch.bfloat16, device=device\n )\n kpe_cache = torch.randn(\n total_pages, page_size, head_dim_kpe, dtype=torch.bfloat16, device=device\n )\n return {\n # plan() signature: qo_indptr, kv_indptr, kv_indices, kv_len_arr,\n # num_heads, head_dim_ckv, head_dim_kpe, page_size, causal, ...\n \"plan\": {\n \"qo_indptr\": qo_indptr,\n \"kv_indptr\": kv_indptr,\n \"kv_indices\": kv_indices,\n \"kv_len_arr\": kv_len,\n \"num_heads\": int(num_qo_heads),\n \"head_dim_ckv\": int(head_dim_ckv),\n \"head_dim_kpe\": int(head_dim_kpe),\n \"page_size\": int(page_size),\n \"causal\": False,\n # tests/attention/test_deepseek_mla.py scales by the pre-absorption\n # QK dimension: qk_nope_head_dim(128) + head_dim_kpe.\n \"sm_scale\": 1.0 / ((128 + head_dim_kpe) ** 0.5),\n \"q_data_type\": torch.bfloat16,\n \"kv_data_type\": torch.bfloat16,\n },\n \"run\": {\n \"q_nope\": q_nope,\n \"q_pe\": q_pe,\n \"ckv_cache\": ckv_cache,\n \"kpe_cache\": kpe_cache,\n },\n }\n" } diff --git a/tests/trace/fi_trace_out/mla_paged_decode_h16_ckv512_kpe64_ps64.json b/tests/trace/fi_trace_out/mla_paged_decode_h16_ckv512_kpe64_ps64.json index 0fa9a6d2c9c..f67a98fb11b 100644 --- a/tests/trace/fi_trace_out/mla_paged_decode_h16_ckv512_kpe64_ps64.json +++ b/tests/trace/fi_trace_out/mla_paged_decode_h16_ckv512_kpe64_ps64.json @@ -156,7 +156,7 @@ "description": "The 2-based log-sum-exp of attention logits." } }, - "reference": "from __future__ import annotations\nimport math\nimport torch\nimport torch.nn.functional as F\n\n@torch.no_grad()\ndef _mla_paged_decode_reference(\n q_nope, q_pe, ckv_cache, kpe_cache, kv_indptr, kv_indices, sm_scale\n):\n batch_size, num_qo_heads, head_dim_ckv = q_nope.shape\n _, _, head_dim_kpe = q_pe.shape\n\n # [num_pages, page_size, head_dim_*] \u2014 keep the page dim; flatten after gather.\n Kc_all = ckv_cache.to(torch.float32)\n Kp_all = kpe_cache.to(torch.float32)\n\n output = torch.zeros(\n (batch_size, num_qo_heads, head_dim_ckv),\n dtype=torch.bfloat16,\n device=q_nope.device,\n )\n lse = torch.full(\n (batch_size, num_qo_heads),\n -float(\"inf\"),\n dtype=torch.float32,\n device=q_nope.device,\n )\n\n for b in range(batch_size):\n page_beg = int(kv_indptr[b].item())\n page_end = int(kv_indptr[b + 1].item())\n if page_beg >= page_end:\n output[b].zero_()\n continue\n # kv_indices are page IDs; gather pages then flatten to a token axis.\n page_ids = kv_indices[page_beg:page_end].to(torch.long)\n Kc = Kc_all[page_ids].reshape(-1, head_dim_ckv) # [L, head_dim_ckv]\n Kp = Kp_all[page_ids].reshape(-1, head_dim_kpe) # [L, head_dim_kpe]\n qn = q_nope[b].to(torch.float32) # [num_qo_heads, head_dim_ckv]\n qp = q_pe[b].to(torch.float32) # [num_qo_heads, head_dim_kpe]\n logits = ((qn @ Kc.T) + (qp @ Kp.T)) * sm_scale # [num_qo_heads, L]\n lse[b] = torch.logsumexp(logits, dim=-1) / math.log(2.0)\n output[b] = (torch.softmax(logits, dim=-1) @ Kc).to(torch.bfloat16)\n\n return output, lse\n", + "reference": "from __future__ import annotations\nimport math\nimport torch\nimport torch.nn.functional as F\n\n@torch.no_grad()\ndef _mla_paged_decode_reference(\n q_nope,\n q_pe,\n ckv_cache,\n kpe_cache,\n kv_indptr,\n kv_indices,\n sm_scale,\n ckv_scale=None,\n kpe_scale=None,\n ckv_scale_arr=None,\n return_lse=False,\n):\n del return_lse\n batch_size, num_qo_heads, head_dim_ckv = q_nope.shape\n _, _, head_dim_kpe = q_pe.shape\n\n # [num_pages, page_size, head_dim_*] \u2014 keep the page dim; flatten after gather.\n Kc_all = ckv_cache.to(torch.float32)\n Kp_all = kpe_cache.to(torch.float32)\n if ckv_cache.dtype == torch.float8_e4m3fn:\n if ckv_scale is None or kpe_scale is None:\n raise ValueError(\"ckv_scale and kpe_scale are required for FP8 KV cache\")\n if ckv_scale_arr is not None:\n Kc_all = (\n Kc_all.reshape(*Kc_all.shape[:-1], head_dim_ckv // 128, 128)\n * ckv_scale_arr.unsqueeze(-1)\n ).reshape_as(Kc_all)\n else:\n Kc_all *= float(ckv_scale)\n Kp_all *= float(kpe_scale)\n\n output = torch.zeros(\n (batch_size, num_qo_heads, head_dim_ckv),\n dtype=torch.bfloat16,\n device=q_nope.device,\n )\n lse = torch.full(\n (batch_size, num_qo_heads),\n -float(\"inf\"),\n dtype=torch.float32,\n device=q_nope.device,\n )\n\n for b in range(batch_size):\n page_beg = int(kv_indptr[b].item())\n page_end = int(kv_indptr[b + 1].item())\n if page_beg >= page_end:\n output[b].zero_()\n continue\n # kv_indices are page IDs; gather pages then flatten to a token axis.\n page_ids = kv_indices[page_beg:page_end].to(torch.long)\n Kc = Kc_all[page_ids].reshape(-1, head_dim_ckv) # [L, head_dim_ckv]\n Kp = Kp_all[page_ids].reshape(-1, head_dim_kpe) # [L, head_dim_kpe]\n qn = q_nope[b].to(torch.float32) # [num_qo_heads, head_dim_ckv]\n qp = q_pe[b].to(torch.float32) # [num_qo_heads, head_dim_kpe]\n logits = ((qn @ Kc.T) + (qp @ Kp.T)) * sm_scale # [num_qo_heads, L]\n lse[b] = torch.logsumexp(logits, dim=-1) / math.log(2.0)\n output[b] = (torch.softmax(logits, dim=-1) @ Kc).to(torch.bfloat16)\n\n return output, lse\n", "check": "def standard_check(\n reference_outputs: Any,\n actual_outputs: Any,\n *,\n rtol: Optional[float] = None,\n atol: Optional[float] = None,\n max_mismatch_pct: float = 0.0,\n min_cos_sim: Optional[float] = 1.0 - 1e-3,\n) -> bool:\n \"\"\"Default trace correctness check used when a template does not override it.\"\"\"\n from flashinfer.trace import default_check\n\n return default_check(\n reference_outputs,\n actual_outputs,\n rtol=rtol,\n atol=atol,\n max_mismatch_pct=max_mismatch_pct,\n min_cos_sim=min_cos_sim,\n )\n", "init": "from __future__ import annotations\nimport math\nimport torch\n\n# ----- shared init helpers -----\n# Copyright (c) 2025 by FlashInfer team.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\n\"\"\"Shared helpers used by ``TraceTemplate.init`` functions.\n\nThis module contains the small set of input-construction patterns that\nrecur across many templates (paged-KV cache index arrays, ragged indptr,\nRoPE pos_ids and cos/sin caches, sampling probs). Each helper is short and\ndocumented; init functions in ``templates/.py`` call into here so\nthe per-template init bodies stay focused on shape/dtype, not boilerplate.\n\nThe full source of this module is **inlined into every dumped JSON's\n``\"init\"`` field** by ``flashinfer/trace/template.py:_render_init_source``,\nso downstream consumers don't need flashinfer installed to re-run the init\nsnippets.\n\"\"\"\n\n\nfrom typing import Optional, Tuple\n\nimport torch\n\n\ndef make_paged_kv_indices(\n batch_size: int,\n num_pages_per_seq: int,\n page_size: int,\n *,\n device: str = \"cuda\",\n) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor]:\n \"\"\"Return ``(kv_indptr, kv_indices, kv_last_page_len)`` for a uniform batch.\n\n Every sequence is assigned exactly ``num_pages_per_seq`` pages, fully\n populated (last-page length == page_size).\n\n Invariants\n ----------\n - ``kv_indptr.shape == (batch_size + 1,)``, dtype int32, monotonic, [0]=0.\n - ``kv_indices == arange(0, batch_size * num_pages_per_seq)``, int32.\n - ``kv_last_page_len == full(batch_size, page_size)``, int32.\n \"\"\"\n total_pages = batch_size * num_pages_per_seq\n kv_indptr = (\n torch.arange(batch_size + 1, dtype=torch.int32, device=device)\n * num_pages_per_seq\n )\n kv_indices = torch.arange(total_pages, dtype=torch.int32, device=device)\n kv_last_page_len = torch.full(\n (batch_size,), page_size, dtype=torch.int32, device=device\n )\n return kv_indptr, kv_indices, kv_last_page_len\n\n\ndef make_ragged_indptr(\n seg_lens,\n *,\n device: str = \"cuda\",\n dtype: torch.dtype = torch.int32,\n) -> torch.Tensor:\n \"\"\"Return cumulative-sum ``indptr`` of length ``len(seg_lens)+1``.\n\n ``seg_lens`` may be a list / tuple / 1-D tensor of segment lengths.\n \"\"\"\n if isinstance(seg_lens, torch.Tensor):\n lens = seg_lens.to(device=device, dtype=dtype)\n else:\n lens = torch.tensor(list(seg_lens), dtype=dtype, device=device)\n indptr = torch.zeros(lens.numel() + 1, dtype=dtype, device=device)\n indptr[1:] = torch.cumsum(lens, dim=0).to(dtype)\n return indptr\n\n\ndef make_uniform_qo_indptr(\n batch_size: int,\n qo_len: int,\n *,\n device: str = \"cuda\",\n) -> torch.Tensor:\n \"\"\"Return ``[0, qo_len, 2*qo_len, ..., batch_size*qo_len]`` int32.\"\"\"\n return torch.arange(batch_size + 1, dtype=torch.int32, device=device) * qo_len\n\n\ndef make_pos_ids(\n nnz: int,\n max_seq_len: Optional[int] = None,\n *,\n device: str = \"cuda\",\n) -> torch.Tensor:\n \"\"\"Return ``[0, 1, ..., nnz-1] (% max_seq_len)`` as int32 on ``device``.\n\n If ``max_seq_len`` is None, no wrapping is applied.\n \"\"\"\n pos = torch.arange(nnz, dtype=torch.int32, device=device)\n if max_seq_len is not None:\n pos = pos % max_seq_len\n return pos\n\n\ndef make_rope_cos_sin_cache(\n max_seq_len: int,\n rope_dim: int,\n *,\n base: float = 1e4,\n device: str = \"cuda\",\n dtype: torch.dtype = torch.float32,\n) -> torch.Tensor:\n \"\"\"Return concatenated ``[cos | sin]`` cache of shape ``[max_seq_len, rope_dim]``.\"\"\"\n t = torch.arange(max_seq_len, dtype=torch.float32, device=device)\n inv = 1.0 / (\n base\n ** (torch.arange(0, rope_dim, 2, dtype=torch.float32, device=device) / rope_dim)\n )\n freqs = t.unsqueeze(-1) * inv.unsqueeze(0)\n cache = torch.cat([torch.cos(freqs), torch.sin(freqs)], dim=-1)\n return cache.to(dtype)\n\n\ndef make_probs(\n batch_size: int,\n vocab_size: int,\n *,\n device: str = \"cuda\",\n dtype: torch.dtype = torch.float32,\n) -> torch.Tensor:\n \"\"\"Return a ``[batch_size, vocab_size]`` probability distribution.\n\n Uses ``softmax(randn(...))`` so each row sums to 1.0. This mirrors the\n pattern used throughout ``tests/utils/test_sampling.py``.\n \"\"\"\n return torch.softmax(\n torch.randn(batch_size, vocab_size, dtype=torch.float32, device=device),\n dim=-1,\n ).to(dtype)\n\n\ndef make_logits(\n batch_size: int,\n vocab_size: int,\n *,\n device: str = \"cuda\",\n dtype: torch.dtype = torch.float32,\n) -> torch.Tensor:\n \"\"\"Return ``randn(batch_size, vocab_size)`` logits.\"\"\"\n return torch.randn(batch_size, vocab_size, dtype=dtype, device=device)\n\n\ndef fp8_safe_randn(\n *shape: int,\n scale: float = 0.1,\n device: str = \"cuda\",\n dtype: torch.dtype = torch.bfloat16,\n) -> torch.Tensor:\n \"\"\"``randn(*shape) * scale`` \u2014 keeps values in the FP8/FP4 representable range.\n\n Tests for fp8/fp4 paths typically multiply ``randn`` by 0.1 to avoid\n saturation when quantizing. Use this helper to mirror that convention.\n \"\"\"\n return (torch.randn(*shape, dtype=dtype, device=device) * scale).to(dtype)\n\n\ndef per_tensor_fp8_quantize(\n x: torch.Tensor,\n *,\n fp8_dtype: torch.dtype = torch.float8_e4m3fn,\n) -> Tuple[torch.Tensor, torch.Tensor]:\n \"\"\"Per-tensor FP8 quantization, mirroring ``tests/utils_fp8.py:to_float8``.\n\n Returns ``(x_fp8, inv_scale)`` where ``inv_scale`` is the dequant\n multiplier (``float \u2248 fp8 * inv_scale``).\n \"\"\"\n finfo = torch.finfo(fp8_dtype)\n amax = x.abs().amax().clamp(min=1e-12)\n scale = finfo.max / amax\n x_q = (x.float() * scale).clamp(min=finfo.min, max=finfo.max).to(fp8_dtype)\n return x_q, scale.float().reciprocal()\n\n\ndef fp8_block_quant_1d(\n x_bf16: torch.Tensor,\n block: int = 128,\n) -> Tuple[torch.Tensor, torch.Tensor]:\n \"\"\"Quantize ``[T, H]`` activations into FP8 with per-``(token, block)``\n column-block scales. Returns ``(x_fp8, scales)`` where\n ``scales`` has shape ``[T, H // block]``.\n\n Mirrors ``_fp8_block_quant_1d`` in\n ``tests/moe/test_dpsk_fused_moe_fp8.py``.\n \"\"\"\n assert x_bf16.dim() == 2\n T, H = x_bf16.shape\n assert H % block == 0\n nb = H // block\n finfo = torch.finfo(torch.float8_e4m3fn)\n max_fp8 = finfo.max\n x_f32 = x_bf16.to(torch.float32)\n x_fp8 = torch.empty((T, H), dtype=torch.float8_e4m3fn, device=x_bf16.device)\n scales = torch.empty((T, nb), dtype=torch.float32, device=x_bf16.device)\n for j in range(nb):\n sl = slice(j * block, (j + 1) * block)\n blk = x_f32[:, sl]\n amax = torch.amax(torch.abs(blk), dim=1)\n s = torch.where(amax > 0, amax / max_fp8, torch.ones_like(amax))\n x_fp8[:, sl] = (blk / s.unsqueeze(1)).to(torch.float8_e4m3fn)\n scales[:, j] = s\n return x_fp8, scales\n\n\ndef fp8_block_quant_2d(\n w_bf16: torch.Tensor,\n block: int = 128,\n) -> Tuple[torch.Tensor, torch.Tensor]:\n \"\"\"Quantize weights ``[..., R, C]`` with 2-D ``block \u00d7 block`` scales.\n\n Returns ``(w_fp8, scales)`` where ``scales`` has shape\n ``[..., R // block, C // block]``. Mirrors ``_fp8_block_quant_2d`` in\n ``tests/moe/test_dpsk_fused_moe_fp8.py``.\n \"\"\"\n assert w_bf16.dim() >= 2\n *prefix, R, C = w_bf16.shape\n assert R % block == 0 and C % block == 0\n nb_r, nb_c = R // block, C // block\n finfo = torch.finfo(torch.float8_e4m3fn)\n max_fp8 = finfo.max\n w_f32 = w_bf16.to(torch.float32).contiguous()\n prefix_ndim = len(prefix)\n reshaped = w_f32.reshape(*prefix, nb_r, block, nb_c, block)\n permute_dims = tuple(range(prefix_ndim)) + (\n prefix_ndim,\n prefix_ndim + 2,\n prefix_ndim + 1,\n prefix_ndim + 3,\n )\n blocks = reshaped.permute(permute_dims).contiguous()\n amax = torch.amax(torch.abs(blocks), dim=(-1, -2))\n scales = torch.where(\n amax > 0, amax / max_fp8, torch.ones_like(amax, dtype=torch.float32)\n )\n q_blocks = (blocks / scales.unsqueeze(-1).unsqueeze(-1)).to(torch.float8_e4m3fn)\n inv_permute = [0] * (prefix_ndim + 4)\n for i, p in enumerate(permute_dims):\n inv_permute[p] = i\n w_fp8 = q_blocks.permute(*inv_permute).reshape(*prefix, R, C).contiguous()\n return w_fp8, scales\n\n\n__all__ = [\n \"make_paged_kv_indices\",\n \"make_ragged_indptr\",\n \"make_uniform_qo_indptr\",\n \"make_pos_ids\",\n \"make_rope_cos_sin_cache\",\n \"make_probs\",\n \"make_logits\",\n \"fp8_safe_randn\",\n \"per_tensor_fp8_quantize\",\n \"fp8_block_quant_1d\",\n \"fp8_block_quant_2d\",\n]\n\n# ----- init -----\ndef _mla_paged_decode_init(\n *,\n batch_size: int,\n num_qo_heads: int = 16,\n head_dim_ckv: int = 512,\n head_dim_kpe: int = 64,\n page_size: int = 64,\n num_pages: int = 0,\n num_pages_per_seq: int = 4,\n len_indptr: int = 0,\n num_kv_indices: int = 0,\n ckv_scale_groups: int = 0,\n device: str = \"cuda\",\n seed: int = 0,\n):\n \"\"\"Build inputs for ``BatchMLAPagedAttentionWrapper.run()``.\n\n Sourced from ``tests/trace/example.py`` MLA section. Default\n ``num_qo_heads=16`` matches DeepSeek-V3 TP=8.\n \"\"\"\n del num_pages, len_indptr, num_kv_indices, ckv_scale_groups\n torch.manual_seed(seed)\n total_pages = batch_size * num_pages_per_seq\n qo_indptr = torch.arange(batch_size + 1, dtype=torch.int32, device=device)\n kv_indptr, kv_indices, _ = make_paged_kv_indices(\n batch_size, num_pages_per_seq, page_size, device=device\n )\n kv_len = torch.full(\n (batch_size,),\n num_pages_per_seq * page_size,\n dtype=torch.int32,\n device=device,\n )\n q_nope = torch.randn(\n batch_size, num_qo_heads, head_dim_ckv, dtype=torch.bfloat16, device=device\n )\n q_pe = torch.randn(\n batch_size, num_qo_heads, head_dim_kpe, dtype=torch.bfloat16, device=device\n )\n ckv_cache = torch.randn(\n total_pages, page_size, head_dim_ckv, dtype=torch.bfloat16, device=device\n )\n kpe_cache = torch.randn(\n total_pages, page_size, head_dim_kpe, dtype=torch.bfloat16, device=device\n )\n return {\n # plan() signature: qo_indptr, kv_indptr, kv_indices, kv_len_arr,\n # num_heads, head_dim_ckv, head_dim_kpe, page_size, causal, ...\n \"plan\": {\n \"qo_indptr\": qo_indptr,\n \"kv_indptr\": kv_indptr,\n \"kv_indices\": kv_indices,\n \"kv_len_arr\": kv_len,\n \"num_heads\": int(num_qo_heads),\n \"head_dim_ckv\": int(head_dim_ckv),\n \"head_dim_kpe\": int(head_dim_kpe),\n \"page_size\": int(page_size),\n \"causal\": False,\n # tests/attention/test_deepseek_mla.py scales by the pre-absorption\n # QK dimension: qk_nope_head_dim(128) + head_dim_kpe.\n \"sm_scale\": 1.0 / ((128 + head_dim_kpe) ** 0.5),\n \"q_data_type\": torch.bfloat16,\n \"kv_data_type\": torch.bfloat16,\n },\n \"run\": {\n \"q_nope\": q_nope,\n \"q_pe\": q_pe,\n \"ckv_cache\": ckv_cache,\n \"kpe_cache\": kpe_cache,\n },\n }\n" } diff --git a/tests/trace/test_fi_trace.py b/tests/trace/test_fi_trace.py index 126e77af978..e4ef98742db 100644 --- a/tests/trace/test_fi_trace.py +++ b/tests/trace/test_fi_trace.py @@ -678,15 +678,23 @@ def test_mla_paged_fi_trace(): q_nope = torch.randn(batch_size, num_qo_heads, head_dim_ckv, dtype=torch.bfloat16) q_pe = torch.randn(batch_size, num_qo_heads, head_dim_kpe, dtype=torch.bfloat16) - ckv_cache = torch.randn(num_pages, page_size, head_dim_ckv, dtype=torch.bfloat16) - kpe_cache = torch.randn(num_pages, page_size, head_dim_kpe, dtype=torch.bfloat16) - ckv_scale_arr = torch.randn(num_pages, page_size, head_dim_ckv // 128) + ckv_cache = torch.zeros( + num_pages, page_size, head_dim_ckv, dtype=torch.float8_e4m3fn + ) + kpe_cache = torch.zeros( + num_pages, page_size, head_dim_kpe, dtype=torch.float8_e4m3fn + ) + ckv_scale_arr = torch.ones( + num_pages, page_size, head_dim_ckv // 128, dtype=torch.float32 + ) defn = BatchMLAPagedAttentionWrapper.run.fi_trace( q_nope=q_nope, q_pe=q_pe, ckv_cache=ckv_cache, kpe_cache=kpe_cache, + ckv_scale=1.0, + kpe_scale=1.0, ckv_scale_arr=ckv_scale_arr, ) _check_defn(defn, "mla_paged", "BatchMLAPagedAttentionWrapper") @@ -696,11 +704,14 @@ def test_mla_paged_fi_trace(): assert axes["head_dim_kpe"]["value"] == head_dim_kpe assert axes["page_size"]["value"] == page_size assert axes["ckv_scale_groups"]["type"] == "var" - assert defn["inputs"]["ckv_scale_arr"]["shape"] == [ + ckv_scale_arr_defn = defn["inputs"]["ckv_scale_arr"] + assert ckv_scale_arr_defn["shape"] == [ "num_pages", "page_size", "ckv_scale_groups", ] + assert ckv_scale_arr_defn["dtype"] == "float32" + assert ckv_scale_arr_defn["optional"] is True # --------------------------------------------------------------------------- diff --git a/tests/trace/test_mla_paged_decode_reference_correctness.py b/tests/trace/test_mla_paged_decode_reference_correctness.py new file mode 100644 index 00000000000..f93b54d15e4 --- /dev/null +++ b/tests/trace/test_mla_paged_decode_reference_correctness.py @@ -0,0 +1,56 @@ +import pytest +import torch + +from flashinfer.trace.templates.attention import mla_paged_decode_trace + + +@pytest.mark.parametrize("use_group_scales", [False, True]) +def test_mla_paged_decode_fp8_reference_dequantization(use_group_scales): + head_dim_ckv = 512 + q_nope = torch.zeros(1, 1, head_dim_ckv, dtype=torch.bfloat16) + q_pe = torch.zeros(1, 1, 64, dtype=torch.bfloat16) + q_pe[..., 0] = 1 + ckv_cache = torch.ones(1, 2, head_dim_ckv, dtype=torch.float8_e4m3fn) + ckv_cache[:, 1] = 2 + kpe_cache = torch.zeros(1, 2, 64, dtype=torch.float8_e4m3fn) + kpe_cache[:, 0, 0] = 1 + kpe_cache[:, 1, 0] = 2 + kv_indptr = torch.tensor([0, 1], dtype=torch.int32) + kv_indices = torch.tensor([0], dtype=torch.int32) + ckv_scale = 7.0 + kpe_scale = 2.0 + ckv_scale_arr = None + dequantized_ckv = ckv_cache.float() * ckv_scale + + if use_group_scales: + ckv_scale_arr = torch.tensor( + [ + [ + [2.0, 3.0, 4.0, 5.0], + [6.0, 7.0, 8.0, 9.0], + ] + ] + ) + dequantized_ckv = ( + ckv_cache.float().reshape(1, 2, 4, 128) * ckv_scale_arr.unsqueeze(-1) + ).reshape_as(dequantized_ckv) + + weights = torch.softmax(torch.tensor([kpe_scale, 2 * kpe_scale]), dim=0) + expected = torch.matmul(weights, dequantized_ckv[0]).reshape(1, 1, head_dim_ckv) + expected = expected.to(torch.bfloat16) + + output, _ = mla_paged_decode_trace.reference( + q_nope, + q_pe, + ckv_cache, + kpe_cache, + kv_indptr, + kv_indices, + 1.0, + ckv_scale, + kpe_scale, + ckv_scale_arr, + False, + ) + + torch.testing.assert_close(output, expected, rtol=0, atol=0) From 2d674f4aa999f17d86015033674464dff76e28a6 Mon Sep 17 00:00:00 2001 From: Xinyuan Tong Date: Fri, 14 Aug 2026 17:07:58 +0000 Subject: [PATCH 5/5] fix: clarify CKV scale API --- flashinfer/mla/_core.py | 48 +++++++++++-------- flashinfer/trace/templates/attention.py | 37 +++++++------- tests/attention/test_deepseek_mla.py | 24 ++++++---- ...mla_paged_decode_h16_ckv512_kpe64_ps1.json | 18 +++---- ...la_paged_decode_h16_ckv512_kpe64_ps64.json | 18 +++---- tests/trace/test_fi_trace.py | 3 +- ..._mla_paged_decode_reference_correctness.py | 3 +- 7 files changed, 85 insertions(+), 66 deletions(-) diff --git a/flashinfer/mla/_core.py b/flashinfer/mla/_core.py index 25ea4afc294..2e39cc52185 100644 --- a/flashinfer/mla/_core.py +++ b/flashinfer/mla/_core.py @@ -2279,8 +2279,8 @@ def run( o_scale: Optional[float] = None, *, ckv_scale: Optional[float] = None, - kpe_scale: Optional[float] = None, ckv_scale_arr: Optional[torch.Tensor] = None, + kpe_scale: Optional[float] = None, ) -> torch.Tensor: ... @overload @@ -2300,8 +2300,8 @@ def run( o_scale: Optional[float] = None, *, ckv_scale: Optional[float] = None, - kpe_scale: Optional[float] = None, ckv_scale_arr: Optional[torch.Tensor] = None, + kpe_scale: Optional[float] = None, ) -> Tuple[torch.Tensor, torch.Tensor]: ... @flashinfer_api(trace=mla_paged_decode_trace) @@ -2321,8 +2321,8 @@ def run( o_scale: Optional[float] = None, *, ckv_scale: Optional[float] = None, - kpe_scale: Optional[float] = None, ckv_scale_arr: Optional[torch.Tensor] = None, + kpe_scale: Optional[float] = None, ) -> Union[torch.Tensor, Tuple[torch.Tensor, torch.Tensor]]: r"""Run the MLA attention computation. @@ -2363,19 +2363,19 @@ def run( the ``cutlass`` backend. ckv_scale : Optional[float] Per-tensor dequantization scale for the compressed-KV cache when - ``kv_data_type`` is FP8 (``real = quantized * ckv_scale``). Required - (together with ``kpe_scale``) for the FP8 KV cache path on the - ``fa2`` or ``fa3`` backend. Must be a finite positive value. Must not be - provided when ``kv_data_type`` is BF16/FP16. - kpe_scale : Optional[float] - Per-tensor dequantization scale for the rope-K cache when - ``kv_data_type`` is FP8 (``real = quantized * kpe_scale``). Same - usage rules as ``ckv_scale``. + ``kv_data_type`` is FP8 (``real = quantized * ckv_scale``). Exactly + one of ``ckv_scale`` or ``ckv_scale_arr`` is required for the FP8 KV + cache path. Must be a finite positive value. Must not be provided + when ``kv_data_type`` is BF16/FP16. ckv_scale_arr : Optional[torch.Tensor] Per-token, per-128-channel CKV dequantization scales. The expected shape is ``ckv_cache.shape[:-1] + (head_dim_ckv // 128,)`` and the - dtype must be contiguous float32. When provided, these values - override ``ckv_scale`` for CKV dequantization. + dtype must be contiguous float32. Exactly one of ``ckv_scale`` or + ``ckv_scale_arr`` is required for the FP8 KV cache path. + kpe_scale : Optional[float] + Per-tensor dequantization scale for the rope-K cache when + ``kv_data_type`` is FP8 (``real = quantized * kpe_scale``). Required + with either CKV scale representation. """ if self._backend == "cutlass": if return_lse: @@ -2467,13 +2467,18 @@ def run( # e4m3fn is the only FP8 dtype reachable here (plan() rejects others). kv_is_fp8 = self._kv_data_type == torch.float8_e4m3fn if kv_is_fp8: - if ckv_scale is None or kpe_scale is None: + if (ckv_scale is None) == (ckv_scale_arr is None): raise ValueError( - "ckv_scale and kpe_scale are required when kv_data_type is FP8." + "Exactly one of ckv_scale or ckv_scale_arr is required when " + "kv_data_type is FP8." ) - ckv_scale_f = float(ckv_scale) + if kpe_scale is None: + raise ValueError("kpe_scale is required when kv_data_type is FP8.") + ckv_scale_f = 1.0 if ckv_scale is None else float(ckv_scale) kpe_scale_f = float(kpe_scale) - if not math.isfinite(ckv_scale_f) or ckv_scale_f <= 0.0: + if ckv_scale is not None and ( + not math.isfinite(ckv_scale_f) or ckv_scale_f <= 0.0 + ): raise ValueError( f"ckv_scale must be a finite positive value, got {ckv_scale}" ) @@ -2482,9 +2487,14 @@ def run( f"kpe_scale must be a finite positive value, got {kpe_scale}" ) else: - if ckv_scale is not None or kpe_scale is not None: + if ( + ckv_scale is not None + or ckv_scale_arr is not None + or kpe_scale is not None + ): raise ValueError( - "ckv_scale / kpe_scale are only valid when kv_data_type is FP8." + "ckv_scale / ckv_scale_arr / kpe_scale are only valid when " + "kv_data_type is FP8." ) ckv_scale_f = 1.0 kpe_scale_f = 1.0 diff --git a/flashinfer/trace/templates/attention.py b/flashinfer/trace/templates/attention.py index 6b088865d29..165d7db0f93 100644 --- a/flashinfer/trace/templates/attention.py +++ b/flashinfer/trace/templates/attention.py @@ -1382,8 +1382,8 @@ def _mla_paged_decode_reference( kv_indices, sm_scale, ckv_scale=None, - kpe_scale=None, ckv_scale_arr=None, + kpe_scale=None, return_lse=False, ): del return_lse @@ -1394,8 +1394,12 @@ def _mla_paged_decode_reference( Kc_all = ckv_cache.to(torch.float32) Kp_all = kpe_cache.to(torch.float32) if ckv_cache.dtype == torch.float8_e4m3fn: - if ckv_scale is None or kpe_scale is None: - raise ValueError("ckv_scale and kpe_scale are required for FP8 KV cache") + if (ckv_scale is None) == (ckv_scale_arr is None): + raise ValueError( + "Exactly one of ckv_scale or ckv_scale_arr is required for FP8 KV cache" + ) + if kpe_scale is None: + raise ValueError("kpe_scale is required for FP8 KV cache") if ckv_scale_arr is not None: Kc_all = ( Kc_all.reshape(*Kc_all.shape[:-1], head_dim_ckv // 128, 128) @@ -1574,28 +1578,27 @@ def _mla_paged_decode_init( optional=True, description=( "Per-tensor dequantization scale for the compressed-KV cache when " - "kv_data_type is FP8 (real = quantized * ckv_scale). Required " - "together with kpe_scale for the FP8 KV cache path on the fa2/fa3 " - "backend. Set during run(), not plan()." + "kv_data_type is FP8 (real = quantized * ckv_scale). Exactly one " + "of ckv_scale or ckv_scale_arr is required for the FP8 KV cache " + "path. Set during run(), not plan()." ), ), - "kpe_scale": Scalar( - "float32", + "ckv_scale_arr": Tensor( + ["num_pages", "page_size", "ckv_scale_groups"], + dtype="float32", optional=True, description=( - "Per-tensor dequantization scale for the rope-K cache when " - "kv_data_type is FP8 (real = quantized * kpe_scale). Required " - "together with ckv_scale for the FP8 KV cache path on the fa2/fa3 " - "backend. Set during run(), not plan()." + "Per-token, per-128-channel CKV dequantization scales for the FP8 " + "KV cache path. Exactly one of ckv_scale or ckv_scale_arr is required." ), ), - "ckv_scale_arr": Tensor( - ["num_pages", "page_size", "ckv_scale_groups"], - dtype="float32", + "kpe_scale": Scalar( + "float32", optional=True, description=( - "Optional per-token, per-128-channel CKV dequantization scales " - "for the FP8 KV cache path." + "Per-tensor dequantization scale for the rope-K cache when " + "kv_data_type is FP8 (real = quantized * kpe_scale). Required " + "with either CKV scale representation. Set during run(), not plan()." ), ), "return_lse": Scalar( diff --git a/tests/attention/test_deepseek_mla.py b/tests/attention/test_deepseek_mla.py index 68581b5c7cb..5ec05d6cf77 100644 --- a/tests/attention/test_deepseek_mla.py +++ b/tests/attention/test_deepseek_mla.py @@ -958,9 +958,8 @@ def test_batch_mla_fp8_nope_group_scales_matches_bf16_reference(backend): sm_scale, torch.bfloat16, torch.float8_e4m3fn, - ckv_scale=1.0, - kpe_scale=1.0, ckv_scale_arr=ckv_scales, + kpe_scale=1.0, head_dim_kpe=0, ) @@ -1409,9 +1408,6 @@ def test_fp8_kv_plan_rejects_fp16_q(): def test_fp8_kv_scales_are_keyword_only(): - """ckv_scale / kpe_scale must be passed by keyword. The `*` marker - in run()'s signature pins this; this test guards against accidental - removal of the marker.""" import inspect sig = inspect.signature(flashinfer.mla.BatchMLAPagedAttentionWrapper.run) @@ -1519,8 +1515,6 @@ def test_fp8_kv_run_rejects_dtype_mismatch(wrong_tensor, wrong_dtype, exc_match) def test_fp8_kv_requires_scales(): - """Forgetting to pass ckv_scale / kpe_scale on the FP8 path should raise - a clear error rather than silently producing wrong output.""" if not is_sm90a_supported(torch.device("cuda")): pytest.skip("FP8 KV path on Hopper MLA requires SM90a") torch.manual_seed(0) @@ -1577,20 +1571,32 @@ def test_fp8_kv_requires_scales(): q_data_type=torch.bfloat16, kv_data_type=torch.float8_e4m3fn, ) - with pytest.raises(ValueError, match="ckv_scale and kpe_scale are required"): + with pytest.raises(ValueError, match="Exactly one of ckv_scale or ckv_scale_arr"): wrapper.run(q_nope, q_pe, ckv_fp8, kpe_fp8) invalid_scales = torch.ones( 1, page_size, HEAD_DIM_CKV // 128 - 1, dtype=torch.float32, device=device ) with pytest.raises(ValueError, match="Invalid shape of ckv_scale_arr"): + wrapper.run( + q_nope, + q_pe, + ckv_fp8, + kpe_fp8, + ckv_scale_arr=invalid_scales, + kpe_scale=1.0, + ) + valid_scales = torch.ones( + 1, page_size, HEAD_DIM_CKV // 128, dtype=torch.float32, device=device + ) + with pytest.raises(ValueError, match="Exactly one of ckv_scale or ckv_scale_arr"): wrapper.run( q_nope, q_pe, ckv_fp8, kpe_fp8, ckv_scale=1.0, + ckv_scale_arr=valid_scales, kpe_scale=1.0, - ckv_scale_arr=invalid_scales, ) diff --git a/tests/trace/fi_trace_out/mla_paged_decode_h16_ckv512_kpe64_ps1.json b/tests/trace/fi_trace_out/mla_paged_decode_h16_ckv512_kpe64_ps1.json index e64b6f38e14..a1621dff2fc 100644 --- a/tests/trace/fi_trace_out/mla_paged_decode_h16_ckv512_kpe64_ps1.json +++ b/tests/trace/fi_trace_out/mla_paged_decode_h16_ckv512_kpe64_ps1.json @@ -112,13 +112,7 @@ "shape": null, "dtype": "float32", "optional": true, - "description": "Per-tensor dequantization scale for the compressed-KV cache when kv_data_type is FP8 (real = quantized * ckv_scale). Required together with kpe_scale for the FP8 KV cache path on the fa2/fa3 backend. Set during run(), not plan()." - }, - "kpe_scale": { - "shape": null, - "dtype": "float32", - "optional": true, - "description": "Per-tensor dequantization scale for the rope-K cache when kv_data_type is FP8 (real = quantized * kpe_scale). Required together with ckv_scale for the FP8 KV cache path on the fa2/fa3 backend. Set during run(), not plan()." + "description": "Per-tensor dequantization scale for the compressed-KV cache when kv_data_type is FP8 (real = quantized * ckv_scale). Exactly one of ckv_scale or ckv_scale_arr is required for the FP8 KV cache path. Set during run(), not plan()." }, "ckv_scale_arr": { "shape": [ @@ -128,7 +122,13 @@ ], "dtype": "float32", "optional": true, - "description": "Optional per-token, per-128-channel CKV dequantization scales for the FP8 KV cache path." + "description": "Per-token, per-128-channel CKV dequantization scales for the FP8 KV cache path. Exactly one of ckv_scale or ckv_scale_arr is required." + }, + "kpe_scale": { + "shape": null, + "dtype": "float32", + "optional": true, + "description": "Per-tensor dequantization scale for the rope-K cache when kv_data_type is FP8 (real = quantized * kpe_scale). Required with either CKV scale representation. Set during run(), not plan()." }, "return_lse": { "shape": null, @@ -156,7 +156,7 @@ "description": "The 2-based log-sum-exp of attention logits." } }, - "reference": "from __future__ import annotations\nimport math\nimport torch\nimport torch.nn.functional as F\n\n@torch.no_grad()\ndef _mla_paged_decode_reference(\n q_nope,\n q_pe,\n ckv_cache,\n kpe_cache,\n kv_indptr,\n kv_indices,\n sm_scale,\n ckv_scale=None,\n kpe_scale=None,\n ckv_scale_arr=None,\n return_lse=False,\n):\n del return_lse\n batch_size, num_qo_heads, head_dim_ckv = q_nope.shape\n _, _, head_dim_kpe = q_pe.shape\n\n # [num_pages, page_size, head_dim_*] \u2014 keep the page dim; flatten after gather.\n Kc_all = ckv_cache.to(torch.float32)\n Kp_all = kpe_cache.to(torch.float32)\n if ckv_cache.dtype == torch.float8_e4m3fn:\n if ckv_scale is None or kpe_scale is None:\n raise ValueError(\"ckv_scale and kpe_scale are required for FP8 KV cache\")\n if ckv_scale_arr is not None:\n Kc_all = (\n Kc_all.reshape(*Kc_all.shape[:-1], head_dim_ckv // 128, 128)\n * ckv_scale_arr.unsqueeze(-1)\n ).reshape_as(Kc_all)\n else:\n Kc_all *= float(ckv_scale)\n Kp_all *= float(kpe_scale)\n\n output = torch.zeros(\n (batch_size, num_qo_heads, head_dim_ckv),\n dtype=torch.bfloat16,\n device=q_nope.device,\n )\n lse = torch.full(\n (batch_size, num_qo_heads),\n -float(\"inf\"),\n dtype=torch.float32,\n device=q_nope.device,\n )\n\n for b in range(batch_size):\n page_beg = int(kv_indptr[b].item())\n page_end = int(kv_indptr[b + 1].item())\n if page_beg >= page_end:\n output[b].zero_()\n continue\n # kv_indices are page IDs; gather pages then flatten to a token axis.\n page_ids = kv_indices[page_beg:page_end].to(torch.long)\n Kc = Kc_all[page_ids].reshape(-1, head_dim_ckv) # [L, head_dim_ckv]\n Kp = Kp_all[page_ids].reshape(-1, head_dim_kpe) # [L, head_dim_kpe]\n qn = q_nope[b].to(torch.float32) # [num_qo_heads, head_dim_ckv]\n qp = q_pe[b].to(torch.float32) # [num_qo_heads, head_dim_kpe]\n logits = ((qn @ Kc.T) + (qp @ Kp.T)) * sm_scale # [num_qo_heads, L]\n lse[b] = torch.logsumexp(logits, dim=-1) / math.log(2.0)\n output[b] = (torch.softmax(logits, dim=-1) @ Kc).to(torch.bfloat16)\n\n return output, lse\n", + "reference": "from __future__ import annotations\nimport math\nimport torch\nimport torch.nn.functional as F\n\n@torch.no_grad()\ndef _mla_paged_decode_reference(\n q_nope,\n q_pe,\n ckv_cache,\n kpe_cache,\n kv_indptr,\n kv_indices,\n sm_scale,\n ckv_scale=None,\n ckv_scale_arr=None,\n kpe_scale=None,\n return_lse=False,\n):\n del return_lse\n batch_size, num_qo_heads, head_dim_ckv = q_nope.shape\n _, _, head_dim_kpe = q_pe.shape\n\n # [num_pages, page_size, head_dim_*] \u2014 keep the page dim; flatten after gather.\n Kc_all = ckv_cache.to(torch.float32)\n Kp_all = kpe_cache.to(torch.float32)\n if ckv_cache.dtype == torch.float8_e4m3fn:\n if (ckv_scale is None) == (ckv_scale_arr is None):\n raise ValueError(\n \"Exactly one of ckv_scale or ckv_scale_arr is required for FP8 KV cache\"\n )\n if kpe_scale is None:\n raise ValueError(\"kpe_scale is required for FP8 KV cache\")\n if ckv_scale_arr is not None:\n Kc_all = (\n Kc_all.reshape(*Kc_all.shape[:-1], head_dim_ckv // 128, 128)\n * ckv_scale_arr.unsqueeze(-1)\n ).reshape_as(Kc_all)\n else:\n Kc_all *= float(ckv_scale)\n Kp_all *= float(kpe_scale)\n\n output = torch.zeros(\n (batch_size, num_qo_heads, head_dim_ckv),\n dtype=torch.bfloat16,\n device=q_nope.device,\n )\n lse = torch.full(\n (batch_size, num_qo_heads),\n -float(\"inf\"),\n dtype=torch.float32,\n device=q_nope.device,\n )\n\n for b in range(batch_size):\n page_beg = int(kv_indptr[b].item())\n page_end = int(kv_indptr[b + 1].item())\n if page_beg >= page_end:\n output[b].zero_()\n continue\n # kv_indices are page IDs; gather pages then flatten to a token axis.\n page_ids = kv_indices[page_beg:page_end].to(torch.long)\n Kc = Kc_all[page_ids].reshape(-1, head_dim_ckv) # [L, head_dim_ckv]\n Kp = Kp_all[page_ids].reshape(-1, head_dim_kpe) # [L, head_dim_kpe]\n qn = q_nope[b].to(torch.float32) # [num_qo_heads, head_dim_ckv]\n qp = q_pe[b].to(torch.float32) # [num_qo_heads, head_dim_kpe]\n logits = ((qn @ Kc.T) + (qp @ Kp.T)) * sm_scale # [num_qo_heads, L]\n lse[b] = torch.logsumexp(logits, dim=-1) / math.log(2.0)\n output[b] = (torch.softmax(logits, dim=-1) @ Kc).to(torch.bfloat16)\n\n return output, lse\n", "check": "def standard_check(\n reference_outputs: Any,\n actual_outputs: Any,\n *,\n rtol: Optional[float] = None,\n atol: Optional[float] = None,\n max_mismatch_pct: float = 0.0,\n min_cos_sim: Optional[float] = 1.0 - 1e-3,\n) -> bool:\n \"\"\"Default trace correctness check used when a template does not override it.\"\"\"\n from flashinfer.trace import default_check\n\n return default_check(\n reference_outputs,\n actual_outputs,\n rtol=rtol,\n atol=atol,\n max_mismatch_pct=max_mismatch_pct,\n min_cos_sim=min_cos_sim,\n )\n", "init": "from __future__ import annotations\nimport math\nimport torch\n\n# ----- shared init helpers -----\n# Copyright (c) 2025 by FlashInfer team.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\n\"\"\"Shared helpers used by ``TraceTemplate.init`` functions.\n\nThis module contains the small set of input-construction patterns that\nrecur across many templates (paged-KV cache index arrays, ragged indptr,\nRoPE pos_ids and cos/sin caches, sampling probs). Each helper is short and\ndocumented; init functions in ``templates/.py`` call into here so\nthe per-template init bodies stay focused on shape/dtype, not boilerplate.\n\nThe full source of this module is **inlined into every dumped JSON's\n``\"init\"`` field** by ``flashinfer/trace/template.py:_render_init_source``,\nso downstream consumers don't need flashinfer installed to re-run the init\nsnippets.\n\"\"\"\n\n\nfrom typing import Optional, Tuple\n\nimport torch\n\n\ndef make_paged_kv_indices(\n batch_size: int,\n num_pages_per_seq: int,\n page_size: int,\n *,\n device: str = \"cuda\",\n) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor]:\n \"\"\"Return ``(kv_indptr, kv_indices, kv_last_page_len)`` for a uniform batch.\n\n Every sequence is assigned exactly ``num_pages_per_seq`` pages, fully\n populated (last-page length == page_size).\n\n Invariants\n ----------\n - ``kv_indptr.shape == (batch_size + 1,)``, dtype int32, monotonic, [0]=0.\n - ``kv_indices == arange(0, batch_size * num_pages_per_seq)``, int32.\n - ``kv_last_page_len == full(batch_size, page_size)``, int32.\n \"\"\"\n total_pages = batch_size * num_pages_per_seq\n kv_indptr = (\n torch.arange(batch_size + 1, dtype=torch.int32, device=device)\n * num_pages_per_seq\n )\n kv_indices = torch.arange(total_pages, dtype=torch.int32, device=device)\n kv_last_page_len = torch.full(\n (batch_size,), page_size, dtype=torch.int32, device=device\n )\n return kv_indptr, kv_indices, kv_last_page_len\n\n\ndef make_ragged_indptr(\n seg_lens,\n *,\n device: str = \"cuda\",\n dtype: torch.dtype = torch.int32,\n) -> torch.Tensor:\n \"\"\"Return cumulative-sum ``indptr`` of length ``len(seg_lens)+1``.\n\n ``seg_lens`` may be a list / tuple / 1-D tensor of segment lengths.\n \"\"\"\n if isinstance(seg_lens, torch.Tensor):\n lens = seg_lens.to(device=device, dtype=dtype)\n else:\n lens = torch.tensor(list(seg_lens), dtype=dtype, device=device)\n indptr = torch.zeros(lens.numel() + 1, dtype=dtype, device=device)\n indptr[1:] = torch.cumsum(lens, dim=0).to(dtype)\n return indptr\n\n\ndef make_uniform_qo_indptr(\n batch_size: int,\n qo_len: int,\n *,\n device: str = \"cuda\",\n) -> torch.Tensor:\n \"\"\"Return ``[0, qo_len, 2*qo_len, ..., batch_size*qo_len]`` int32.\"\"\"\n return torch.arange(batch_size + 1, dtype=torch.int32, device=device) * qo_len\n\n\ndef make_pos_ids(\n nnz: int,\n max_seq_len: Optional[int] = None,\n *,\n device: str = \"cuda\",\n) -> torch.Tensor:\n \"\"\"Return ``[0, 1, ..., nnz-1] (% max_seq_len)`` as int32 on ``device``.\n\n If ``max_seq_len`` is None, no wrapping is applied.\n \"\"\"\n pos = torch.arange(nnz, dtype=torch.int32, device=device)\n if max_seq_len is not None:\n pos = pos % max_seq_len\n return pos\n\n\ndef make_rope_cos_sin_cache(\n max_seq_len: int,\n rope_dim: int,\n *,\n base: float = 1e4,\n device: str = \"cuda\",\n dtype: torch.dtype = torch.float32,\n) -> torch.Tensor:\n \"\"\"Return concatenated ``[cos | sin]`` cache of shape ``[max_seq_len, rope_dim]``.\"\"\"\n t = torch.arange(max_seq_len, dtype=torch.float32, device=device)\n inv = 1.0 / (\n base\n ** (torch.arange(0, rope_dim, 2, dtype=torch.float32, device=device) / rope_dim)\n )\n freqs = t.unsqueeze(-1) * inv.unsqueeze(0)\n cache = torch.cat([torch.cos(freqs), torch.sin(freqs)], dim=-1)\n return cache.to(dtype)\n\n\ndef make_probs(\n batch_size: int,\n vocab_size: int,\n *,\n device: str = \"cuda\",\n dtype: torch.dtype = torch.float32,\n) -> torch.Tensor:\n \"\"\"Return a ``[batch_size, vocab_size]`` probability distribution.\n\n Uses ``softmax(randn(...))`` so each row sums to 1.0. This mirrors the\n pattern used throughout ``tests/utils/test_sampling.py``.\n \"\"\"\n return torch.softmax(\n torch.randn(batch_size, vocab_size, dtype=torch.float32, device=device),\n dim=-1,\n ).to(dtype)\n\n\ndef make_logits(\n batch_size: int,\n vocab_size: int,\n *,\n device: str = \"cuda\",\n dtype: torch.dtype = torch.float32,\n) -> torch.Tensor:\n \"\"\"Return ``randn(batch_size, vocab_size)`` logits.\"\"\"\n return torch.randn(batch_size, vocab_size, dtype=dtype, device=device)\n\n\ndef fp8_safe_randn(\n *shape: int,\n scale: float = 0.1,\n device: str = \"cuda\",\n dtype: torch.dtype = torch.bfloat16,\n) -> torch.Tensor:\n \"\"\"``randn(*shape) * scale`` \u2014 keeps values in the FP8/FP4 representable range.\n\n Tests for fp8/fp4 paths typically multiply ``randn`` by 0.1 to avoid\n saturation when quantizing. Use this helper to mirror that convention.\n \"\"\"\n return (torch.randn(*shape, dtype=dtype, device=device) * scale).to(dtype)\n\n\ndef per_tensor_fp8_quantize(\n x: torch.Tensor,\n *,\n fp8_dtype: torch.dtype = torch.float8_e4m3fn,\n) -> Tuple[torch.Tensor, torch.Tensor]:\n \"\"\"Per-tensor FP8 quantization, mirroring ``tests/utils_fp8.py:to_float8``.\n\n Returns ``(x_fp8, inv_scale)`` where ``inv_scale`` is the dequant\n multiplier (``float \u2248 fp8 * inv_scale``).\n \"\"\"\n finfo = torch.finfo(fp8_dtype)\n amax = x.abs().amax().clamp(min=1e-12)\n scale = finfo.max / amax\n x_q = (x.float() * scale).clamp(min=finfo.min, max=finfo.max).to(fp8_dtype)\n return x_q, scale.float().reciprocal()\n\n\ndef fp8_block_quant_1d(\n x_bf16: torch.Tensor,\n block: int = 128,\n) -> Tuple[torch.Tensor, torch.Tensor]:\n \"\"\"Quantize ``[T, H]`` activations into FP8 with per-``(token, block)``\n column-block scales. Returns ``(x_fp8, scales)`` where\n ``scales`` has shape ``[T, H // block]``.\n\n Mirrors ``_fp8_block_quant_1d`` in\n ``tests/moe/test_dpsk_fused_moe_fp8.py``.\n \"\"\"\n assert x_bf16.dim() == 2\n T, H = x_bf16.shape\n assert H % block == 0\n nb = H // block\n finfo = torch.finfo(torch.float8_e4m3fn)\n max_fp8 = finfo.max\n x_f32 = x_bf16.to(torch.float32)\n x_fp8 = torch.empty((T, H), dtype=torch.float8_e4m3fn, device=x_bf16.device)\n scales = torch.empty((T, nb), dtype=torch.float32, device=x_bf16.device)\n for j in range(nb):\n sl = slice(j * block, (j + 1) * block)\n blk = x_f32[:, sl]\n amax = torch.amax(torch.abs(blk), dim=1)\n s = torch.where(amax > 0, amax / max_fp8, torch.ones_like(amax))\n x_fp8[:, sl] = (blk / s.unsqueeze(1)).to(torch.float8_e4m3fn)\n scales[:, j] = s\n return x_fp8, scales\n\n\ndef fp8_block_quant_2d(\n w_bf16: torch.Tensor,\n block: int = 128,\n) -> Tuple[torch.Tensor, torch.Tensor]:\n \"\"\"Quantize weights ``[..., R, C]`` with 2-D ``block \u00d7 block`` scales.\n\n Returns ``(w_fp8, scales)`` where ``scales`` has shape\n ``[..., R // block, C // block]``. Mirrors ``_fp8_block_quant_2d`` in\n ``tests/moe/test_dpsk_fused_moe_fp8.py``.\n \"\"\"\n assert w_bf16.dim() >= 2\n *prefix, R, C = w_bf16.shape\n assert R % block == 0 and C % block == 0\n nb_r, nb_c = R // block, C // block\n finfo = torch.finfo(torch.float8_e4m3fn)\n max_fp8 = finfo.max\n w_f32 = w_bf16.to(torch.float32).contiguous()\n prefix_ndim = len(prefix)\n reshaped = w_f32.reshape(*prefix, nb_r, block, nb_c, block)\n permute_dims = tuple(range(prefix_ndim)) + (\n prefix_ndim,\n prefix_ndim + 2,\n prefix_ndim + 1,\n prefix_ndim + 3,\n )\n blocks = reshaped.permute(permute_dims).contiguous()\n amax = torch.amax(torch.abs(blocks), dim=(-1, -2))\n scales = torch.where(\n amax > 0, amax / max_fp8, torch.ones_like(amax, dtype=torch.float32)\n )\n q_blocks = (blocks / scales.unsqueeze(-1).unsqueeze(-1)).to(torch.float8_e4m3fn)\n inv_permute = [0] * (prefix_ndim + 4)\n for i, p in enumerate(permute_dims):\n inv_permute[p] = i\n w_fp8 = q_blocks.permute(*inv_permute).reshape(*prefix, R, C).contiguous()\n return w_fp8, scales\n\n\n__all__ = [\n \"make_paged_kv_indices\",\n \"make_ragged_indptr\",\n \"make_uniform_qo_indptr\",\n \"make_pos_ids\",\n \"make_rope_cos_sin_cache\",\n \"make_probs\",\n \"make_logits\",\n \"fp8_safe_randn\",\n \"per_tensor_fp8_quantize\",\n \"fp8_block_quant_1d\",\n \"fp8_block_quant_2d\",\n]\n\n# ----- init -----\ndef _mla_paged_decode_init(\n *,\n batch_size: int,\n num_qo_heads: int = 16,\n head_dim_ckv: int = 512,\n head_dim_kpe: int = 64,\n page_size: int = 64,\n num_pages: int = 0,\n num_pages_per_seq: int = 4,\n len_indptr: int = 0,\n num_kv_indices: int = 0,\n ckv_scale_groups: int = 0,\n device: str = \"cuda\",\n seed: int = 0,\n):\n \"\"\"Build inputs for ``BatchMLAPagedAttentionWrapper.run()``.\n\n Sourced from ``tests/trace/example.py`` MLA section. Default\n ``num_qo_heads=16`` matches DeepSeek-V3 TP=8.\n \"\"\"\n del num_pages, len_indptr, num_kv_indices, ckv_scale_groups\n torch.manual_seed(seed)\n total_pages = batch_size * num_pages_per_seq\n qo_indptr = torch.arange(batch_size + 1, dtype=torch.int32, device=device)\n kv_indptr, kv_indices, _ = make_paged_kv_indices(\n batch_size, num_pages_per_seq, page_size, device=device\n )\n kv_len = torch.full(\n (batch_size,),\n num_pages_per_seq * page_size,\n dtype=torch.int32,\n device=device,\n )\n q_nope = torch.randn(\n batch_size, num_qo_heads, head_dim_ckv, dtype=torch.bfloat16, device=device\n )\n q_pe = torch.randn(\n batch_size, num_qo_heads, head_dim_kpe, dtype=torch.bfloat16, device=device\n )\n ckv_cache = torch.randn(\n total_pages, page_size, head_dim_ckv, dtype=torch.bfloat16, device=device\n )\n kpe_cache = torch.randn(\n total_pages, page_size, head_dim_kpe, dtype=torch.bfloat16, device=device\n )\n return {\n # plan() signature: qo_indptr, kv_indptr, kv_indices, kv_len_arr,\n # num_heads, head_dim_ckv, head_dim_kpe, page_size, causal, ...\n \"plan\": {\n \"qo_indptr\": qo_indptr,\n \"kv_indptr\": kv_indptr,\n \"kv_indices\": kv_indices,\n \"kv_len_arr\": kv_len,\n \"num_heads\": int(num_qo_heads),\n \"head_dim_ckv\": int(head_dim_ckv),\n \"head_dim_kpe\": int(head_dim_kpe),\n \"page_size\": int(page_size),\n \"causal\": False,\n # tests/attention/test_deepseek_mla.py scales by the pre-absorption\n # QK dimension: qk_nope_head_dim(128) + head_dim_kpe.\n \"sm_scale\": 1.0 / ((128 + head_dim_kpe) ** 0.5),\n \"q_data_type\": torch.bfloat16,\n \"kv_data_type\": torch.bfloat16,\n },\n \"run\": {\n \"q_nope\": q_nope,\n \"q_pe\": q_pe,\n \"ckv_cache\": ckv_cache,\n \"kpe_cache\": kpe_cache,\n },\n }\n" } diff --git a/tests/trace/fi_trace_out/mla_paged_decode_h16_ckv512_kpe64_ps64.json b/tests/trace/fi_trace_out/mla_paged_decode_h16_ckv512_kpe64_ps64.json index f67a98fb11b..7e9c18856e4 100644 --- a/tests/trace/fi_trace_out/mla_paged_decode_h16_ckv512_kpe64_ps64.json +++ b/tests/trace/fi_trace_out/mla_paged_decode_h16_ckv512_kpe64_ps64.json @@ -112,13 +112,7 @@ "shape": null, "dtype": "float32", "optional": true, - "description": "Per-tensor dequantization scale for the compressed-KV cache when kv_data_type is FP8 (real = quantized * ckv_scale). Required together with kpe_scale for the FP8 KV cache path on the fa2/fa3 backend. Set during run(), not plan()." - }, - "kpe_scale": { - "shape": null, - "dtype": "float32", - "optional": true, - "description": "Per-tensor dequantization scale for the rope-K cache when kv_data_type is FP8 (real = quantized * kpe_scale). Required together with ckv_scale for the FP8 KV cache path on the fa2/fa3 backend. Set during run(), not plan()." + "description": "Per-tensor dequantization scale for the compressed-KV cache when kv_data_type is FP8 (real = quantized * ckv_scale). Exactly one of ckv_scale or ckv_scale_arr is required for the FP8 KV cache path. Set during run(), not plan()." }, "ckv_scale_arr": { "shape": [ @@ -128,7 +122,13 @@ ], "dtype": "float32", "optional": true, - "description": "Optional per-token, per-128-channel CKV dequantization scales for the FP8 KV cache path." + "description": "Per-token, per-128-channel CKV dequantization scales for the FP8 KV cache path. Exactly one of ckv_scale or ckv_scale_arr is required." + }, + "kpe_scale": { + "shape": null, + "dtype": "float32", + "optional": true, + "description": "Per-tensor dequantization scale for the rope-K cache when kv_data_type is FP8 (real = quantized * kpe_scale). Required with either CKV scale representation. Set during run(), not plan()." }, "return_lse": { "shape": null, @@ -156,7 +156,7 @@ "description": "The 2-based log-sum-exp of attention logits." } }, - "reference": "from __future__ import annotations\nimport math\nimport torch\nimport torch.nn.functional as F\n\n@torch.no_grad()\ndef _mla_paged_decode_reference(\n q_nope,\n q_pe,\n ckv_cache,\n kpe_cache,\n kv_indptr,\n kv_indices,\n sm_scale,\n ckv_scale=None,\n kpe_scale=None,\n ckv_scale_arr=None,\n return_lse=False,\n):\n del return_lse\n batch_size, num_qo_heads, head_dim_ckv = q_nope.shape\n _, _, head_dim_kpe = q_pe.shape\n\n # [num_pages, page_size, head_dim_*] \u2014 keep the page dim; flatten after gather.\n Kc_all = ckv_cache.to(torch.float32)\n Kp_all = kpe_cache.to(torch.float32)\n if ckv_cache.dtype == torch.float8_e4m3fn:\n if ckv_scale is None or kpe_scale is None:\n raise ValueError(\"ckv_scale and kpe_scale are required for FP8 KV cache\")\n if ckv_scale_arr is not None:\n Kc_all = (\n Kc_all.reshape(*Kc_all.shape[:-1], head_dim_ckv // 128, 128)\n * ckv_scale_arr.unsqueeze(-1)\n ).reshape_as(Kc_all)\n else:\n Kc_all *= float(ckv_scale)\n Kp_all *= float(kpe_scale)\n\n output = torch.zeros(\n (batch_size, num_qo_heads, head_dim_ckv),\n dtype=torch.bfloat16,\n device=q_nope.device,\n )\n lse = torch.full(\n (batch_size, num_qo_heads),\n -float(\"inf\"),\n dtype=torch.float32,\n device=q_nope.device,\n )\n\n for b in range(batch_size):\n page_beg = int(kv_indptr[b].item())\n page_end = int(kv_indptr[b + 1].item())\n if page_beg >= page_end:\n output[b].zero_()\n continue\n # kv_indices are page IDs; gather pages then flatten to a token axis.\n page_ids = kv_indices[page_beg:page_end].to(torch.long)\n Kc = Kc_all[page_ids].reshape(-1, head_dim_ckv) # [L, head_dim_ckv]\n Kp = Kp_all[page_ids].reshape(-1, head_dim_kpe) # [L, head_dim_kpe]\n qn = q_nope[b].to(torch.float32) # [num_qo_heads, head_dim_ckv]\n qp = q_pe[b].to(torch.float32) # [num_qo_heads, head_dim_kpe]\n logits = ((qn @ Kc.T) + (qp @ Kp.T)) * sm_scale # [num_qo_heads, L]\n lse[b] = torch.logsumexp(logits, dim=-1) / math.log(2.0)\n output[b] = (torch.softmax(logits, dim=-1) @ Kc).to(torch.bfloat16)\n\n return output, lse\n", + "reference": "from __future__ import annotations\nimport math\nimport torch\nimport torch.nn.functional as F\n\n@torch.no_grad()\ndef _mla_paged_decode_reference(\n q_nope,\n q_pe,\n ckv_cache,\n kpe_cache,\n kv_indptr,\n kv_indices,\n sm_scale,\n ckv_scale=None,\n ckv_scale_arr=None,\n kpe_scale=None,\n return_lse=False,\n):\n del return_lse\n batch_size, num_qo_heads, head_dim_ckv = q_nope.shape\n _, _, head_dim_kpe = q_pe.shape\n\n # [num_pages, page_size, head_dim_*] \u2014 keep the page dim; flatten after gather.\n Kc_all = ckv_cache.to(torch.float32)\n Kp_all = kpe_cache.to(torch.float32)\n if ckv_cache.dtype == torch.float8_e4m3fn:\n if (ckv_scale is None) == (ckv_scale_arr is None):\n raise ValueError(\n \"Exactly one of ckv_scale or ckv_scale_arr is required for FP8 KV cache\"\n )\n if kpe_scale is None:\n raise ValueError(\"kpe_scale is required for FP8 KV cache\")\n if ckv_scale_arr is not None:\n Kc_all = (\n Kc_all.reshape(*Kc_all.shape[:-1], head_dim_ckv // 128, 128)\n * ckv_scale_arr.unsqueeze(-1)\n ).reshape_as(Kc_all)\n else:\n Kc_all *= float(ckv_scale)\n Kp_all *= float(kpe_scale)\n\n output = torch.zeros(\n (batch_size, num_qo_heads, head_dim_ckv),\n dtype=torch.bfloat16,\n device=q_nope.device,\n )\n lse = torch.full(\n (batch_size, num_qo_heads),\n -float(\"inf\"),\n dtype=torch.float32,\n device=q_nope.device,\n )\n\n for b in range(batch_size):\n page_beg = int(kv_indptr[b].item())\n page_end = int(kv_indptr[b + 1].item())\n if page_beg >= page_end:\n output[b].zero_()\n continue\n # kv_indices are page IDs; gather pages then flatten to a token axis.\n page_ids = kv_indices[page_beg:page_end].to(torch.long)\n Kc = Kc_all[page_ids].reshape(-1, head_dim_ckv) # [L, head_dim_ckv]\n Kp = Kp_all[page_ids].reshape(-1, head_dim_kpe) # [L, head_dim_kpe]\n qn = q_nope[b].to(torch.float32) # [num_qo_heads, head_dim_ckv]\n qp = q_pe[b].to(torch.float32) # [num_qo_heads, head_dim_kpe]\n logits = ((qn @ Kc.T) + (qp @ Kp.T)) * sm_scale # [num_qo_heads, L]\n lse[b] = torch.logsumexp(logits, dim=-1) / math.log(2.0)\n output[b] = (torch.softmax(logits, dim=-1) @ Kc).to(torch.bfloat16)\n\n return output, lse\n", "check": "def standard_check(\n reference_outputs: Any,\n actual_outputs: Any,\n *,\n rtol: Optional[float] = None,\n atol: Optional[float] = None,\n max_mismatch_pct: float = 0.0,\n min_cos_sim: Optional[float] = 1.0 - 1e-3,\n) -> bool:\n \"\"\"Default trace correctness check used when a template does not override it.\"\"\"\n from flashinfer.trace import default_check\n\n return default_check(\n reference_outputs,\n actual_outputs,\n rtol=rtol,\n atol=atol,\n max_mismatch_pct=max_mismatch_pct,\n min_cos_sim=min_cos_sim,\n )\n", "init": "from __future__ import annotations\nimport math\nimport torch\n\n# ----- shared init helpers -----\n# Copyright (c) 2025 by FlashInfer team.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\n\"\"\"Shared helpers used by ``TraceTemplate.init`` functions.\n\nThis module contains the small set of input-construction patterns that\nrecur across many templates (paged-KV cache index arrays, ragged indptr,\nRoPE pos_ids and cos/sin caches, sampling probs). Each helper is short and\ndocumented; init functions in ``templates/.py`` call into here so\nthe per-template init bodies stay focused on shape/dtype, not boilerplate.\n\nThe full source of this module is **inlined into every dumped JSON's\n``\"init\"`` field** by ``flashinfer/trace/template.py:_render_init_source``,\nso downstream consumers don't need flashinfer installed to re-run the init\nsnippets.\n\"\"\"\n\n\nfrom typing import Optional, Tuple\n\nimport torch\n\n\ndef make_paged_kv_indices(\n batch_size: int,\n num_pages_per_seq: int,\n page_size: int,\n *,\n device: str = \"cuda\",\n) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor]:\n \"\"\"Return ``(kv_indptr, kv_indices, kv_last_page_len)`` for a uniform batch.\n\n Every sequence is assigned exactly ``num_pages_per_seq`` pages, fully\n populated (last-page length == page_size).\n\n Invariants\n ----------\n - ``kv_indptr.shape == (batch_size + 1,)``, dtype int32, monotonic, [0]=0.\n - ``kv_indices == arange(0, batch_size * num_pages_per_seq)``, int32.\n - ``kv_last_page_len == full(batch_size, page_size)``, int32.\n \"\"\"\n total_pages = batch_size * num_pages_per_seq\n kv_indptr = (\n torch.arange(batch_size + 1, dtype=torch.int32, device=device)\n * num_pages_per_seq\n )\n kv_indices = torch.arange(total_pages, dtype=torch.int32, device=device)\n kv_last_page_len = torch.full(\n (batch_size,), page_size, dtype=torch.int32, device=device\n )\n return kv_indptr, kv_indices, kv_last_page_len\n\n\ndef make_ragged_indptr(\n seg_lens,\n *,\n device: str = \"cuda\",\n dtype: torch.dtype = torch.int32,\n) -> torch.Tensor:\n \"\"\"Return cumulative-sum ``indptr`` of length ``len(seg_lens)+1``.\n\n ``seg_lens`` may be a list / tuple / 1-D tensor of segment lengths.\n \"\"\"\n if isinstance(seg_lens, torch.Tensor):\n lens = seg_lens.to(device=device, dtype=dtype)\n else:\n lens = torch.tensor(list(seg_lens), dtype=dtype, device=device)\n indptr = torch.zeros(lens.numel() + 1, dtype=dtype, device=device)\n indptr[1:] = torch.cumsum(lens, dim=0).to(dtype)\n return indptr\n\n\ndef make_uniform_qo_indptr(\n batch_size: int,\n qo_len: int,\n *,\n device: str = \"cuda\",\n) -> torch.Tensor:\n \"\"\"Return ``[0, qo_len, 2*qo_len, ..., batch_size*qo_len]`` int32.\"\"\"\n return torch.arange(batch_size + 1, dtype=torch.int32, device=device) * qo_len\n\n\ndef make_pos_ids(\n nnz: int,\n max_seq_len: Optional[int] = None,\n *,\n device: str = \"cuda\",\n) -> torch.Tensor:\n \"\"\"Return ``[0, 1, ..., nnz-1] (% max_seq_len)`` as int32 on ``device``.\n\n If ``max_seq_len`` is None, no wrapping is applied.\n \"\"\"\n pos = torch.arange(nnz, dtype=torch.int32, device=device)\n if max_seq_len is not None:\n pos = pos % max_seq_len\n return pos\n\n\ndef make_rope_cos_sin_cache(\n max_seq_len: int,\n rope_dim: int,\n *,\n base: float = 1e4,\n device: str = \"cuda\",\n dtype: torch.dtype = torch.float32,\n) -> torch.Tensor:\n \"\"\"Return concatenated ``[cos | sin]`` cache of shape ``[max_seq_len, rope_dim]``.\"\"\"\n t = torch.arange(max_seq_len, dtype=torch.float32, device=device)\n inv = 1.0 / (\n base\n ** (torch.arange(0, rope_dim, 2, dtype=torch.float32, device=device) / rope_dim)\n )\n freqs = t.unsqueeze(-1) * inv.unsqueeze(0)\n cache = torch.cat([torch.cos(freqs), torch.sin(freqs)], dim=-1)\n return cache.to(dtype)\n\n\ndef make_probs(\n batch_size: int,\n vocab_size: int,\n *,\n device: str = \"cuda\",\n dtype: torch.dtype = torch.float32,\n) -> torch.Tensor:\n \"\"\"Return a ``[batch_size, vocab_size]`` probability distribution.\n\n Uses ``softmax(randn(...))`` so each row sums to 1.0. This mirrors the\n pattern used throughout ``tests/utils/test_sampling.py``.\n \"\"\"\n return torch.softmax(\n torch.randn(batch_size, vocab_size, dtype=torch.float32, device=device),\n dim=-1,\n ).to(dtype)\n\n\ndef make_logits(\n batch_size: int,\n vocab_size: int,\n *,\n device: str = \"cuda\",\n dtype: torch.dtype = torch.float32,\n) -> torch.Tensor:\n \"\"\"Return ``randn(batch_size, vocab_size)`` logits.\"\"\"\n return torch.randn(batch_size, vocab_size, dtype=dtype, device=device)\n\n\ndef fp8_safe_randn(\n *shape: int,\n scale: float = 0.1,\n device: str = \"cuda\",\n dtype: torch.dtype = torch.bfloat16,\n) -> torch.Tensor:\n \"\"\"``randn(*shape) * scale`` \u2014 keeps values in the FP8/FP4 representable range.\n\n Tests for fp8/fp4 paths typically multiply ``randn`` by 0.1 to avoid\n saturation when quantizing. Use this helper to mirror that convention.\n \"\"\"\n return (torch.randn(*shape, dtype=dtype, device=device) * scale).to(dtype)\n\n\ndef per_tensor_fp8_quantize(\n x: torch.Tensor,\n *,\n fp8_dtype: torch.dtype = torch.float8_e4m3fn,\n) -> Tuple[torch.Tensor, torch.Tensor]:\n \"\"\"Per-tensor FP8 quantization, mirroring ``tests/utils_fp8.py:to_float8``.\n\n Returns ``(x_fp8, inv_scale)`` where ``inv_scale`` is the dequant\n multiplier (``float \u2248 fp8 * inv_scale``).\n \"\"\"\n finfo = torch.finfo(fp8_dtype)\n amax = x.abs().amax().clamp(min=1e-12)\n scale = finfo.max / amax\n x_q = (x.float() * scale).clamp(min=finfo.min, max=finfo.max).to(fp8_dtype)\n return x_q, scale.float().reciprocal()\n\n\ndef fp8_block_quant_1d(\n x_bf16: torch.Tensor,\n block: int = 128,\n) -> Tuple[torch.Tensor, torch.Tensor]:\n \"\"\"Quantize ``[T, H]`` activations into FP8 with per-``(token, block)``\n column-block scales. Returns ``(x_fp8, scales)`` where\n ``scales`` has shape ``[T, H // block]``.\n\n Mirrors ``_fp8_block_quant_1d`` in\n ``tests/moe/test_dpsk_fused_moe_fp8.py``.\n \"\"\"\n assert x_bf16.dim() == 2\n T, H = x_bf16.shape\n assert H % block == 0\n nb = H // block\n finfo = torch.finfo(torch.float8_e4m3fn)\n max_fp8 = finfo.max\n x_f32 = x_bf16.to(torch.float32)\n x_fp8 = torch.empty((T, H), dtype=torch.float8_e4m3fn, device=x_bf16.device)\n scales = torch.empty((T, nb), dtype=torch.float32, device=x_bf16.device)\n for j in range(nb):\n sl = slice(j * block, (j + 1) * block)\n blk = x_f32[:, sl]\n amax = torch.amax(torch.abs(blk), dim=1)\n s = torch.where(amax > 0, amax / max_fp8, torch.ones_like(amax))\n x_fp8[:, sl] = (blk / s.unsqueeze(1)).to(torch.float8_e4m3fn)\n scales[:, j] = s\n return x_fp8, scales\n\n\ndef fp8_block_quant_2d(\n w_bf16: torch.Tensor,\n block: int = 128,\n) -> Tuple[torch.Tensor, torch.Tensor]:\n \"\"\"Quantize weights ``[..., R, C]`` with 2-D ``block \u00d7 block`` scales.\n\n Returns ``(w_fp8, scales)`` where ``scales`` has shape\n ``[..., R // block, C // block]``. Mirrors ``_fp8_block_quant_2d`` in\n ``tests/moe/test_dpsk_fused_moe_fp8.py``.\n \"\"\"\n assert w_bf16.dim() >= 2\n *prefix, R, C = w_bf16.shape\n assert R % block == 0 and C % block == 0\n nb_r, nb_c = R // block, C // block\n finfo = torch.finfo(torch.float8_e4m3fn)\n max_fp8 = finfo.max\n w_f32 = w_bf16.to(torch.float32).contiguous()\n prefix_ndim = len(prefix)\n reshaped = w_f32.reshape(*prefix, nb_r, block, nb_c, block)\n permute_dims = tuple(range(prefix_ndim)) + (\n prefix_ndim,\n prefix_ndim + 2,\n prefix_ndim + 1,\n prefix_ndim + 3,\n )\n blocks = reshaped.permute(permute_dims).contiguous()\n amax = torch.amax(torch.abs(blocks), dim=(-1, -2))\n scales = torch.where(\n amax > 0, amax / max_fp8, torch.ones_like(amax, dtype=torch.float32)\n )\n q_blocks = (blocks / scales.unsqueeze(-1).unsqueeze(-1)).to(torch.float8_e4m3fn)\n inv_permute = [0] * (prefix_ndim + 4)\n for i, p in enumerate(permute_dims):\n inv_permute[p] = i\n w_fp8 = q_blocks.permute(*inv_permute).reshape(*prefix, R, C).contiguous()\n return w_fp8, scales\n\n\n__all__ = [\n \"make_paged_kv_indices\",\n \"make_ragged_indptr\",\n \"make_uniform_qo_indptr\",\n \"make_pos_ids\",\n \"make_rope_cos_sin_cache\",\n \"make_probs\",\n \"make_logits\",\n \"fp8_safe_randn\",\n \"per_tensor_fp8_quantize\",\n \"fp8_block_quant_1d\",\n \"fp8_block_quant_2d\",\n]\n\n# ----- init -----\ndef _mla_paged_decode_init(\n *,\n batch_size: int,\n num_qo_heads: int = 16,\n head_dim_ckv: int = 512,\n head_dim_kpe: int = 64,\n page_size: int = 64,\n num_pages: int = 0,\n num_pages_per_seq: int = 4,\n len_indptr: int = 0,\n num_kv_indices: int = 0,\n ckv_scale_groups: int = 0,\n device: str = \"cuda\",\n seed: int = 0,\n):\n \"\"\"Build inputs for ``BatchMLAPagedAttentionWrapper.run()``.\n\n Sourced from ``tests/trace/example.py`` MLA section. Default\n ``num_qo_heads=16`` matches DeepSeek-V3 TP=8.\n \"\"\"\n del num_pages, len_indptr, num_kv_indices, ckv_scale_groups\n torch.manual_seed(seed)\n total_pages = batch_size * num_pages_per_seq\n qo_indptr = torch.arange(batch_size + 1, dtype=torch.int32, device=device)\n kv_indptr, kv_indices, _ = make_paged_kv_indices(\n batch_size, num_pages_per_seq, page_size, device=device\n )\n kv_len = torch.full(\n (batch_size,),\n num_pages_per_seq * page_size,\n dtype=torch.int32,\n device=device,\n )\n q_nope = torch.randn(\n batch_size, num_qo_heads, head_dim_ckv, dtype=torch.bfloat16, device=device\n )\n q_pe = torch.randn(\n batch_size, num_qo_heads, head_dim_kpe, dtype=torch.bfloat16, device=device\n )\n ckv_cache = torch.randn(\n total_pages, page_size, head_dim_ckv, dtype=torch.bfloat16, device=device\n )\n kpe_cache = torch.randn(\n total_pages, page_size, head_dim_kpe, dtype=torch.bfloat16, device=device\n )\n return {\n # plan() signature: qo_indptr, kv_indptr, kv_indices, kv_len_arr,\n # num_heads, head_dim_ckv, head_dim_kpe, page_size, causal, ...\n \"plan\": {\n \"qo_indptr\": qo_indptr,\n \"kv_indptr\": kv_indptr,\n \"kv_indices\": kv_indices,\n \"kv_len_arr\": kv_len,\n \"num_heads\": int(num_qo_heads),\n \"head_dim_ckv\": int(head_dim_ckv),\n \"head_dim_kpe\": int(head_dim_kpe),\n \"page_size\": int(page_size),\n \"causal\": False,\n # tests/attention/test_deepseek_mla.py scales by the pre-absorption\n # QK dimension: qk_nope_head_dim(128) + head_dim_kpe.\n \"sm_scale\": 1.0 / ((128 + head_dim_kpe) ** 0.5),\n \"q_data_type\": torch.bfloat16,\n \"kv_data_type\": torch.bfloat16,\n },\n \"run\": {\n \"q_nope\": q_nope,\n \"q_pe\": q_pe,\n \"ckv_cache\": ckv_cache,\n \"kpe_cache\": kpe_cache,\n },\n }\n" } diff --git a/tests/trace/test_fi_trace.py b/tests/trace/test_fi_trace.py index e4ef98742db..3967511e9e7 100644 --- a/tests/trace/test_fi_trace.py +++ b/tests/trace/test_fi_trace.py @@ -693,9 +693,8 @@ def test_mla_paged_fi_trace(): q_pe=q_pe, ckv_cache=ckv_cache, kpe_cache=kpe_cache, - ckv_scale=1.0, - kpe_scale=1.0, ckv_scale_arr=ckv_scale_arr, + kpe_scale=1.0, ) _check_defn(defn, "mla_paged", "BatchMLAPagedAttentionWrapper") axes = defn["axes"] diff --git a/tests/trace/test_mla_paged_decode_reference_correctness.py b/tests/trace/test_mla_paged_decode_reference_correctness.py index f93b54d15e4..df623caa35c 100644 --- a/tests/trace/test_mla_paged_decode_reference_correctness.py +++ b/tests/trace/test_mla_paged_decode_reference_correctness.py @@ -23,6 +23,7 @@ def test_mla_paged_decode_fp8_reference_dequantization(use_group_scales): dequantized_ckv = ckv_cache.float() * ckv_scale if use_group_scales: + ckv_scale = None ckv_scale_arr = torch.tensor( [ [ @@ -48,8 +49,8 @@ def test_mla_paged_decode_fp8_reference_dequantization(use_group_scales): kv_indices, 1.0, ckv_scale, - kpe_scale, ckv_scale_arr, + kpe_scale, False, )