From c19c36a1c2f2c066f3fdb334d8304795d6594912 Mon Sep 17 00:00:00 2001 From: mingyangw Date: Tue, 25 Aug 2026 23:56:35 +0000 Subject: [PATCH 1/6] perf(attention): specialize paged FA2 equal-stride KV Select equal- and independent-stride paged FA2 variants at dispatch time so the equal-stride kernel reuses K offsets for V. Preserve unequal-stride and asymmetric NVFP4 behavior, with focused decode, prefill, and JIT specialization coverage. --- csrc/batch_prefill.cu | 19 +++-- csrc/batch_prefill_paged_kernel_inst.jinja | 5 +- include/flashinfer/attention/prefill.cuh | 60 ++++++++++------ tests/attention/test_batch_prefill_kernels.py | 63 ++++++++++++++++ tests/attention/test_tensor_cores_decode.py | 71 +++++++++++++++++++ tests/jit/test_attention_utils.py | 55 ++++++++++++++ 6 files changed, 244 insertions(+), 29 deletions(-) diff --git a/csrc/batch_prefill.cu b/csrc/batch_prefill.cu index 1f083268452..7d48de07fc6 100644 --- a/csrc/batch_prefill.cu +++ b/csrc/batch_prefill.cu @@ -23,7 +23,7 @@ namespace flashinfer { -template cudaError_t BatchPrefillWithPagedKVCacheDispatched(Params params, typename Params::DTypeO* tmp_v, @@ -279,6 +279,10 @@ void BatchPrefillWithPagedKVCacheRun(TensorView float_workspace_buffer, auto k_cache_strides = paged_k_cache.strides(); auto v_cache_strides = paged_v_cache.strides(); TVM_FFI_ICHECK_EQ(paged_k_cache.ndim(), paged_v_cache.ndim()); + bool kv_strides_are_identical = true; + for (int i = 0; i < paged_k_cache.ndim(); ++i) { + kv_strides_are_identical &= paged_k_cache.stride(i) == paged_v_cache.stride(i); + } ffi::CUDADeviceGuard device_guard(float_workspace_buffer.device().device_id); const cudaStream_t stream = get_stream(float_workspace_buffer.device()); @@ -359,11 +363,14 @@ void BatchPrefillWithPagedKVCacheRun(TensorView float_workspace_buffer, cudaError_t status = cudaSuccess; - DISPATCH_CTA_TILE_Q(plan_info.cta_tile_q, CTA_TILE_Q, { - status = flashinfer::BatchPrefillWithPagedKVCacheDispatched< - CTA_TILE_Q, HEAD_DIM_QK, HEAD_DIM_VO, POS_ENCODING_MODE, - /*use_fp16_qk_reduction=*/USE_FP16_QK_REDUCTION, MASK_MODE, AttentionVariant, - PagedParams>(params, tmp_v, tmp_s, enable_pdl, stream); + DISPATCH_BOOL(kv_strides_are_identical, SAME_KV_STRIDES, [&] { + DISPATCH_CTA_TILE_Q(plan_info.cta_tile_q, CTA_TILE_Q, { + status = flashinfer::BatchPrefillWithPagedKVCacheDispatched< + /*SAME_KV_STRIDES=*/SAME_KV_STRIDES, CTA_TILE_Q, HEAD_DIM_QK, HEAD_DIM_VO, + POS_ENCODING_MODE, /*use_fp16_qk_reduction=*/USE_FP16_QK_REDUCTION, MASK_MODE, + AttentionVariant, PagedParams>(params, tmp_v, tmp_s, enable_pdl, stream); + }); + return true; }); TVM_FFI_ICHECK(status == cudaSuccess) diff --git a/csrc/batch_prefill_paged_kernel_inst.jinja b/csrc/batch_prefill_paged_kernel_inst.jinja index 5518d8b71b5..ff9682c0d2d 100644 --- a/csrc/batch_prefill_paged_kernel_inst.jinja +++ b/csrc/batch_prefill_paged_kernel_inst.jinja @@ -9,10 +9,13 @@ constexpr auto use_custom_mask = {{ mask_mode }} == MaskMode::kCustom; {16} for head_dim_qk >= 512 with a smaller head_dim_vo (o_frag register pressure permits only CTA16 there), and {16, 64, 128} otherwise; don't instantiate unreachable variants. #} +{% for same_kv_strides in ["true", "false"] %} {% for cta_tile_q in ([16, 32] if head_dim_vo | int >= 512 else ([16] if head_dim_qk | int >= 512 else [16, 64, 128])) %} template cudaError_t BatchPrefillWithPagedKVCacheDispatched< - /*CTA_TILE_Q=*/{{cta_tile_q}}, {{head_dim_qk}}, {{head_dim_vo}}, {{pos_encoding_mode}}, {{use_fp16_qk_reduction}}, {{mask_mode}}, + /*SAME_KV_STRIDES=*/{{same_kv_strides}}, /*CTA_TILE_Q=*/{{cta_tile_q}}, + {{head_dim_qk}}, {{head_dim_vo}}, {{pos_encoding_mode}}, {{use_fp16_qk_reduction}}, {{mask_mode}}, {{ variant_name }}, PagedParams>(PagedParams params, {{ dtype_o }}* tmp_v, float* tmp_s, bool enable_pdl, cudaStream_t stream); {% endfor %} +{% endfor %} }; // namespace flashinfer diff --git a/include/flashinfer/attention/prefill.cuh b/include/flashinfer/attention/prefill.cuh index 34e6cd62a60..8d17c467472 100644 --- a/include/flashinfer/attention/prefill.cuh +++ b/include/flashinfer/attention/prefill.cuh @@ -3796,7 +3796,7 @@ __device__ __forceinline__ void vosplit_write_o( } } -template +template __device__ __forceinline__ void BatchPrefillWithPagedKVCacheDevice( const Params params, SmemStorage& smem_storage, const dim3 tid = threadIdx, const uint32_t bx = blockIdx.x, const uint32_t kv_head_idx = blockIdx.z, @@ -4010,8 +4010,9 @@ __device__ __forceinline__ void BatchPrefillWithPagedKVCacheDevice( // smem path computes offsets on the fly, so the arrays collapse to [1] stubs there. [[maybe_unused]] size_t thr_local_kv_offset_k[KTraits::USE_KV_SHARED_SMEM ? 1 : NUM_PAGED_KV_OFFSETS]; - [[maybe_unused]] size_t - thr_local_kv_offset_v[KTraits::USE_KV_SHARED_SMEM ? 1 : NUM_PAGED_KV_OFFSETS]; + [[maybe_unused]] size_t thr_local_kv_offset_v[KTraits::USE_KV_SHARED_SMEM || SAME_KV_STRIDES + ? 1 + : NUM_PAGED_KV_OFFSETS]; uint32_t k_smem_offset_r = k_smem.template get_permuted_offset( get_warp_idx_kv(tid.z) * NUM_MMA_KV * 16 + 8 * (lane_idx / 16) + @@ -4070,8 +4071,10 @@ __device__ __forceinline__ void BatchPrefillWithPagedKVCacheDevice( (lane_idx % KV_THR_LAYOUT_COL) * upcast_size() / fp4_pack_factor; thr_local_kv_offset_k[i] = paged_kv.protective_get_k_offset( page_iter, kv_head_idx, entry_idx, feat_idx, last_indptr); - thr_local_kv_offset_v[i] = paged_kv.protective_get_v_offset( - page_iter, kv_head_idx, entry_idx, feat_idx, last_indptr); + if constexpr (!SAME_KV_STRIDES) { + thr_local_kv_offset_v[i] = paged_kv.protective_get_v_offset( + page_iter, kv_head_idx, entry_idx, feat_idx, last_indptr); + } } page_produce_kv(&smem_storage, &k_smem_offset_w, paged_kv.k_data, 0, thr_local_kv_offset_k, chunk_size, warp_idx, lane_idx); @@ -4083,8 +4086,13 @@ __device__ __forceinline__ void BatchPrefillWithPagedKVCacheDevice( cp_async::commit_group(); // Shared K/V loads V(0) inside iter 0 after Q.K^T; preloading it would clobber K(0). if constexpr (!KTraits::USE_KV_SHARED_SMEM) { - page_produce_kv(&smem_storage, &v_smem_offset_w, paged_kv.v_data, 0, - thr_local_kv_offset_v, chunk_size, warp_idx, lane_idx); + if constexpr (SAME_KV_STRIDES) { + page_produce_kv(&smem_storage, &v_smem_offset_w, paged_kv.v_data, 0, + thr_local_kv_offset_k, chunk_size, warp_idx, lane_idx); + } else { + page_produce_kv(&smem_storage, &v_smem_offset_w, paged_kv.v_data, 0, + thr_local_kv_offset_v, chunk_size, warp_idx, lane_idx); + } page_produce_kv_sf(&smem_storage, maybe_v_cache_sf, packed_page_iter_base, last_indptr * (uint32_t)paged_kv.page_size, kv_head_idx, v_sf_stride_page, v_sf_stride_h, v_sf_stride_n, @@ -4177,8 +4185,10 @@ __device__ __forceinline__ void BatchPrefillWithPagedKVCacheDevice( (lane_idx % KV_THR_LAYOUT_COL) * upcast_size() / fp4_pack_factor; thr_local_kv_offset_k[i] = paged_kv.protective_get_k_offset( page_iter, kv_head_idx, entry_idx, feat_idx, last_indptr); - thr_local_kv_offset_v[i] = paged_kv.protective_get_v_offset( - page_iter, kv_head_idx, entry_idx, feat_idx, last_indptr); + if constexpr (!SAME_KV_STRIDES) { + thr_local_kv_offset_v[i] = paged_kv.protective_get_v_offset( + page_iter, kv_head_idx, entry_idx, feat_idx, last_indptr); + } } } // Shared K/V serializes loads (no K/V prefetch overlap) -> drain fully. @@ -4331,9 +4341,15 @@ __device__ __forceinline__ void BatchPrefillWithPagedKVCacheDevice( cp_async::commit_group(); packed_page_iter_base = next_packed_page_iter_base; } else { - page_produce_kv(&smem_storage, &v_smem_offset_w, paged_kv.v_data, - (iter + 1) * CTA_TILE_KV, thr_local_kv_offset_v, - chunk_size, warp_idx, lane_idx); + if constexpr (SAME_KV_STRIDES) { + page_produce_kv(&smem_storage, &v_smem_offset_w, paged_kv.v_data, + (iter + 1) * CTA_TILE_KV, thr_local_kv_offset_k, + chunk_size, warp_idx, lane_idx); + } else { + page_produce_kv(&smem_storage, &v_smem_offset_w, paged_kv.v_data, + (iter + 1) * CTA_TILE_KV, thr_local_kv_offset_v, + chunk_size, warp_idx, lane_idx); + } page_produce_kv_sf( &smem_storage, maybe_v_cache_sf, packed_page_iter_base, last_indptr * (uint32_t)paged_kv.page_size, kv_head_idx, v_sf_stride_page, @@ -4412,12 +4428,12 @@ __device__ __forceinline__ void BatchPrefillWithPagedKVCacheDevice( #endif } -template +template __global__ __launch_bounds__(KTraits::NUM_THREADS) void BatchPrefillWithPagedKVCacheKernel( const __grid_constant__ Params params) { extern __shared__ uint8_t smem[]; auto& smem_storage = reinterpret_cast(smem); - BatchPrefillWithPagedKVCacheDevice(params, smem_storage); + BatchPrefillWithPagedKVCacheDevice(params, smem_storage); } template cudaError_t BatchPrefillWithPagedKVCacheDispatchedImpl(Params params, @@ -4767,7 +4783,7 @@ cudaError_t BatchPrefillWithPagedKVCacheDispatchedImpl(Params params, FLASHINFER_ERROR(err_msg.str()); } else { size_t smem_size = sizeof(typename KTraits::SharedStoragePaged); - auto kernel = BatchPrefillWithPagedKVCacheKernel; + auto kernel = BatchPrefillWithPagedKVCacheKernel; // Exact final check: the analytic NUM_MMA_KV budget can slightly // under-count the real struct (cross-warp merge buffers, padding); // fail with a clear error instead of a launch-time cudaErrorInvalidValue. @@ -4834,17 +4850,17 @@ cudaError_t BatchPrefillWithPagedKVCacheDispatchedImpl(Params params, return cudaSuccess; } -template cudaError_t BatchPrefillWithPagedKVCacheDispatched(Params params, typename Params::DTypeO* tmp_v, float* tmp_s, bool enable_pdl, cudaStream_t stream) { -#define FLASHINFER_PAGED_PREFILL_CALL(EN) \ - (BatchPrefillWithPagedKVCacheDispatchedImpl(params, tmp_v, tmp_s, \ - enable_pdl, stream)) +#define FLASHINFER_PAGED_PREFILL_CALL(EN) \ + (BatchPrefillWithPagedKVCacheDispatchedImpl< \ + SAME_KV_STRIDES, CTA_TILE_Q, HEAD_DIM_QK, HEAD_DIM_VO, POS_ENCODING_MODE, \ + USE_FP16_QK_REDUCTION, MASK_MODE, AttentionVariant, Params, EN>(params, tmp_v, tmp_s, \ + enable_pdl, stream)) FLASHINFER_DISPATCH_FP4_REPACK(typename Params::DTypeKV, CTA_TILE_Q, HEAD_DIM_QK, HEAD_DIM_VO, FLASHINFER_PAGED_PREFILL_CALL) #undef FLASHINFER_PAGED_PREFILL_CALL diff --git a/tests/attention/test_batch_prefill_kernels.py b/tests/attention/test_batch_prefill_kernels.py index 5804fcb3328..7eb441668a4 100644 --- a/tests/attention/test_batch_prefill_kernels.py +++ b/tests/attention/test_batch_prefill_kernels.py @@ -338,6 +338,69 @@ def test_batch_prefill_with_paged_kv_cache( _assert_no_ref_mismatch(mismatch_counts) +@pytest.mark.cuda +def test_batch_prefill_equal_kv_strides_bf16(): + """Exercise the equal-stride FA2 paged path for the GPT-OSS head shape.""" + torch.manual_seed(42) + qo_len = kv_len = 65 + page_size = 16 + num_qo_heads, num_kv_heads, head_dim = 64, 8, 64 + pages_per_request = (kv_len + page_size - 1) // page_size + + q = torch.randn( + qo_len, + num_qo_heads, + head_dim, + device="cuda", + dtype=torch.bfloat16, + ) + k = torch.randn( + pages_per_request, + page_size, + num_kv_heads, + head_dim, + device="cuda", + dtype=torch.bfloat16, + ) + v = torch.randn_like(k) + assert k.stride() == v.stride() + + qo_indptr = torch.tensor([0, qo_len], device="cuda", dtype=torch.int32) + kv_indptr = torch.tensor([0, pages_per_request], device="cuda", dtype=torch.int32) + kv_indices = torch.arange(pages_per_request, device="cuda", dtype=torch.int32) + last_page_len = torch.tensor( + [(kv_len - 1) % page_size + 1], device="cuda", dtype=torch.int32 + ) + + workspace = torch.empty(128 * 1024 * 1024, device="cuda", dtype=torch.uint8) + wrapper = flashinfer.BatchPrefillWithPagedKVCacheWrapper( + workspace, "NHD", backend="fa2" + ) + wrapper.plan( + qo_indptr, + kv_indptr, + kv_indices, + last_page_len, + num_qo_heads, + num_kv_heads, + head_dim, + page_size, + causal=True, + q_data_type=torch.bfloat16, + kv_data_type=torch.bfloat16, + ) + actual = wrapper.run(q, (k, v)) + expected = flashinfer.prefill.single_prefill_with_kv_cache( + q, + k.view(-1, num_kv_heads, head_dim)[:kv_len], + v.view(-1, num_kv_heads, head_dim)[:kv_len], + causal=True, + backend="fa2", + ) + + torch.testing.assert_close(actual.float(), expected.float(), rtol=1e-2, atol=1e-2) + + @pytest.mark.parametrize("causal", [False, True]) @pytest.mark.parametrize("pos_encoding_mode", ["NONE", "ROPE_LLAMA"]) def test_batch_prefill_with_paged_kv_cache_head_dim_512( diff --git a/tests/attention/test_tensor_cores_decode.py b/tests/attention/test_tensor_cores_decode.py index 19db15a640f..39e57c14c13 100644 --- a/tests/attention/test_tensor_cores_decode.py +++ b/tests/attention/test_tensor_cores_decode.py @@ -193,6 +193,77 @@ def test_batch_decode_tensor_cores( torch.testing.assert_close(lse, lse_tensor_cores, rtol=1e-3, atol=1e-3) +@pytest.mark.cuda +def test_batch_decode_tensor_cores_equal_kv_strides_bf16_group_size_5(): + """Exercise the equal-stride FA2 paged path for the Llama-4 head ratio.""" + torch.manual_seed(42) + batch_size, kv_len, page_size = 2, 97, 16 + num_qo_heads, num_kv_heads, head_dim = 40, 8, 128 + pages_per_request = (kv_len + page_size - 1) // page_size + total_pages = batch_size * pages_per_request + + q = torch.randn( + batch_size, + num_qo_heads, + head_dim, + device="cuda", + dtype=torch.bfloat16, + ) + k = torch.randn( + total_pages, + page_size, + num_kv_heads, + head_dim, + device="cuda", + dtype=torch.bfloat16, + ) + v = torch.randn_like(k) + assert k.stride() == v.stride() + + kv_indptr = ( + torch.arange(batch_size + 1, device="cuda", dtype=torch.int32) + * pages_per_request + ) + kv_indices = torch.arange(total_pages, device="cuda", dtype=torch.int32) + last_page_len = torch.full( + (batch_size,), + (kv_len - 1) % page_size + 1, + device="cuda", + dtype=torch.int32, + ) + + workspace = torch.empty(128 * 1024 * 1024, device="cuda", dtype=torch.uint8) + wrapper = flashinfer.BatchDecodeWithPagedKVCacheWrapper( + workspace, "NHD", use_tensor_cores=True, backend="fa2" + ) + wrapper.plan( + kv_indptr, + kv_indices, + last_page_len, + num_qo_heads, + num_kv_heads, + head_dim, + page_size, + q_data_type=torch.bfloat16, + kv_data_type=torch.bfloat16, + ) + actual = wrapper.run(q, (k, v)) + + k_dense = k.view(batch_size, pages_per_request * page_size, num_kv_heads, head_dim)[ + :, :kv_len + ] + v_dense = v.view(batch_size, pages_per_request * page_size, num_kv_heads, head_dim)[ + :, :kv_len + ] + group_size = num_qo_heads // num_kv_heads + k_dense = k_dense.repeat_interleave(group_size, dim=2).float() + v_dense = v_dense.repeat_interleave(group_size, dim=2).float() + logits = torch.einsum("bhd,bkhd->bhk", q.float(), k_dense) * head_dim**-0.5 + expected = torch.einsum("bhk,bkhd->bhd", torch.softmax(logits, dim=-1), v_dense) + + torch.testing.assert_close(actual.float(), expected, rtol=1e-2, atol=1e-2) + + @pytest.mark.parametrize("batch_size", [12, 17]) @pytest.mark.parametrize("kv_len", [54, 97, 512]) @pytest.mark.parametrize("page_size", [1, 8, 16]) diff --git a/tests/jit/test_attention_utils.py b/tests/jit/test_attention_utils.py index 4ddc0672efb..62f31e7ece3 100644 --- a/tests/jit/test_attention_utils.py +++ b/tests/jit/test_attention_utils.py @@ -2,7 +2,9 @@ import torch +from flashinfer.jit import core from flashinfer.jit import env as jit_env +from flashinfer.jit.attention import modules as attention_modules from flashinfer.jit.attention.modules import gen_customize_batch_prefill_module @@ -55,6 +57,59 @@ def test_batch_prefill_nvfp4_swa_paged_params_declares_sf_strides( assert f"uint32_t {field}_stride_n;" in generated +def test_batch_prefill_generates_equal_and_independent_stride_specializations( + tmp_path, monkeypatch +): + repo_root = Path(__file__).resolve().parents[2] + monkeypatch.setattr(core, "check_cuda_arch", lambda: None) + monkeypatch.setattr( + attention_modules.current_compilation_context, + "TARGET_CUDA_ARCHS", + {(10, 3)}, + ) + monkeypatch.setattr(jit_env, "FLASHINFER_GEN_SRC_DIR", tmp_path / "generated") + monkeypatch.setattr(jit_env, "FLASHINFER_CSRC_DIR", repo_root / "csrc") + + uri = "test_batch_prefill_stride_specializations" + spec = gen_customize_batch_prefill_module( + "fa2", + uri, + torch.bfloat16, + torch.bfloat16, + torch.bfloat16, + torch.int32, + 64, + 64, + [], + [], + [], + [], + "DefaultAttention", + "struct DefaultAttention {};", + ) + + assert spec.name == uri + paged = tmp_path / "generated" / uri / "batch_prefill_paged_kernel_mask_0.cu" + ragged = tmp_path / "generated" / uri / "batch_prefill_ragged_kernel_mask_0.cu" + binding = tmp_path / "generated" / uri / "batch_prefill.cu" + paged_text = paged.read_text() + assert ( + paged_text.count("template cudaError_t BatchPrefillWithPagedKVCacheDispatched<") + == 6 + ) + assert paged_text.count("/*SAME_KV_STRIDES=*/true") == 3 + assert paged_text.count("/*SAME_KV_STRIDES=*/false") == 3 + assert ( + ragged.read_text().count( + "template cudaError_t BatchPrefillWithRaggedKVCacheDispatched<" + ) + == 3 + ) + assert ( + "DISPATCH_BOOL(kv_strides_are_identical, SAME_KV_STRIDES" in binding.read_text() + ) + + def test_batch_prefill_nvfp4_requires_sf_tensors(): try: gen_customize_batch_prefill_module( From d9ce45113789e577a77e40d02007428af0191c02 Mon Sep 17 00:00:00 2001 From: mingyangw Date: Thu, 10 Sep 2026 03:23:12 +0000 Subject: [PATCH 2/6] perf(attention): load independent KV stride kernels on demand --- csrc/batch_pod.cu | 22 +- csrc/batch_prefill.cu | 143 +------ csrc/batch_prefill_customize_config.jinja | 5 + csrc/batch_prefill_paged.cu | 40 ++ csrc/batch_prefill_paged.cuh | 181 +++++++++ csrc/batch_prefill_paged_jit_binding.cu | 32 ++ csrc/batch_prefill_paged_kernel_inst.jinja | 2 +- csrc/pod.cu | 11 +- flashinfer/aot.py | 3 +- flashinfer/attention/_core.py | 2 + flashinfer/cascade.py | 11 + flashinfer/decode.py | 28 ++ flashinfer/jit/attention/modules.py | 254 +++++++++++- flashinfer/prefill.py | 118 +++++- tests/attention/test_batch_prefill_kernels.py | 383 ++++++++++++++++++ tests/attention/test_block_sparse.py | 45 ++ tests/attention/test_shared_prefix_kernels.py | 101 +++++ tests/attention/test_tensor_cores_decode.py | 110 +++++ tests/jit/test_attention_utils.py | 305 ++++++++++++-- tests/jit/test_batch_prefill_lazy_loader.py | 309 ++++++++++++++ tests/test_helpers/paged_kv.py | 73 ++++ ...est_batch_pod_run_reference_correctness.py | 88 ++++ ...aged_kv_cache_run_reference_correctness.py | 80 ++++ ..._block_sparse_run_reference_correctness.py | 82 ++++ 24 files changed, 2213 insertions(+), 215 deletions(-) create mode 100644 csrc/batch_prefill_paged.cu create mode 100644 csrc/batch_prefill_paged.cuh create mode 100644 csrc/batch_prefill_paged_jit_binding.cu create mode 100644 tests/jit/test_batch_prefill_lazy_loader.py create mode 100644 tests/test_helpers/paged_kv.py diff --git a/csrc/batch_pod.cu b/csrc/batch_pod.cu index 98ff9d83dae..8114d9b9eaf 100644 --- a/csrc/batch_pod.cu +++ b/csrc/batch_pod.cu @@ -90,15 +90,10 @@ void batch_pod_with_kv_cache_tensor( const auto q_stride_n_p = q_p.stride(0); const auto q_stride_h_p = q_p.stride(1); - // get kv_cache_strides - const int64_t* kv_cache_strides_p = nullptr; + // get independent K/V cache strides auto k_strides_p = paged_k_cache_p.strides(); auto v_strides_p = paged_v_cache_p.strides(); TVM_FFI_ICHECK_EQ(k_strides_p.size(), v_strides_p.size()); - for (int i = 0; i < k_strides_p.size(); ++i) { - TVM_FFI_ICHECK_EQ(k_strides_p[i], v_strides_p[i]); - } - kv_cache_strides_p = k_strides_p.data(); ffi::CUDADeviceGuard device_guard(float_workspace_buffer_p.device().device_id); const cudaStream_t stream = get_stream(float_workspace_buffer_p.device()); @@ -141,15 +136,10 @@ void batch_pod_with_kv_cache_tensor( const auto q_stride_n_d = q_d.stride(0); const auto q_stride_h_d = q_d.stride(1); - // get kv_cache_strides - const int64_t* kv_cache_strides_d = nullptr; + // get independent K/V cache strides auto k_strides_d = paged_k_cache_d.strides(); auto v_strides_d = paged_v_cache_d.strides(); TVM_FFI_ICHECK_EQ(k_strides_d.size(), v_strides_d.size()); - for (int i = 0; i < k_strides_d.size(); ++i) { - TVM_FFI_ICHECK_EQ(k_strides_d[i], v_strides_d[i]); - } - kv_cache_strides_d = k_strides_d.data(); // Already handled by prefill // ffi::CUDADeviceGuard device_guard(float_workspace_buffer_d.device().device_id); @@ -167,8 +157,8 @@ void batch_pod_with_kv_cache_tensor( paged_kv_t paged_kv( num_kv_heads_p, page_size_p, HEAD_DIM_VO, batch_size_p, kv_layout_p, static_cast(paged_k_cache_p.data_ptr()), - static_cast(paged_v_cache_p.data_ptr()), kv_cache_strides_p, - static_cast(paged_kv_indices_p.data_ptr()), + static_cast(paged_v_cache_p.data_ptr()), k_strides_p.data(), + v_strides_p.data(), static_cast(paged_kv_indices_p.data_ptr()), static_cast(paged_kv_indptr_p.data_ptr()), static_cast(paged_kv_last_page_len_p.data_ptr())); params.paged_kv = paged_kv; @@ -245,8 +235,8 @@ void batch_pod_with_kv_cache_tensor( paged_kv_t paged_kv( num_kv_heads_d, page_size_d, HEAD_DIM_VO, batch_size_d, kv_layout_d, static_cast(paged_k_cache_d.data_ptr()), - static_cast(paged_v_cache_d.data_ptr()), kv_cache_strides_d, - static_cast(paged_kv_indices_d.data_ptr()), + static_cast(paged_v_cache_d.data_ptr()), k_strides_d.data(), + v_strides_d.data(), static_cast(paged_kv_indices_d.data_ptr()), static_cast(paged_kv_indptr_d.data_ptr()), static_cast(paged_kv_last_page_len_d.data_ptr())); params.paged_kv = paged_kv; diff --git a/csrc/batch_prefill.cu b/csrc/batch_prefill.cu index 7d48de07fc6..a7cfaaf46bb 100644 --- a/csrc/batch_prefill.cu +++ b/csrc/batch_prefill.cu @@ -236,145 +236,4 @@ void BatchPrefillWithRaggedKVCacheRun(TensorView float_workspace_buffer, }); } -void BatchPrefillWithPagedKVCacheRun(TensorView float_workspace_buffer, - TensorView int_workspace_buffer, Array plan_info_vec, - TensorView q, TensorView paged_k_cache, - TensorView paged_v_cache, TensorView qo_indptr, - TensorView paged_kv_indptr, TensorView paged_kv_indices, - TensorView paged_kv_last_page_len, TensorView o, - Optional maybe_lse, int64_t mask_mode_code, - int64_t layout, int64_t window_left, - bool enable_pdl ADDITIONAL_FUNC_PARAMS) { - PrefillPlanInfo plan_info; - plan_info.FromVector(std::vector(plan_info_vec.begin(), plan_info_vec.end())); - QKVLayout kv_layout = static_cast(layout); - int64_t batch_size = paged_kv_indptr.size(0) - 1; - int64_t num_qo_heads = q.size(1); - int64_t num_kv_heads, page_size; - uint32_t head_dim_qk = q.size(2); - if (kv_layout == QKVLayout::kHND) { - num_kv_heads = paged_k_cache.size(1); - page_size = paged_k_cache.size(2); - } else { - page_size = paged_k_cache.size(1); - num_kv_heads = paged_k_cache.size(2); - } - - if (maybe_lse) { - const auto& lse = *maybe_lse; - TVM_FFI_ICHECK_EQ(lse.size(0), q.size(0)); - TVM_FFI_ICHECK_EQ(lse.size(1), q.size(1)); - } - - void* float_buffer_ptr = static_cast(float_workspace_buffer.data_ptr()); - void* int_buffer_ptr = static_cast(int_workspace_buffer.data_ptr()); - - const MaskMode mask_mode = static_cast(mask_mode_code); - - // get q_stride_n and q_stride_h - const auto q_stride_n = q.stride(0); - const auto q_stride_h = q.stride(1); - - // get kv-cache strides - auto k_cache_strides = paged_k_cache.strides(); - auto v_cache_strides = paged_v_cache.strides(); - TVM_FFI_ICHECK_EQ(paged_k_cache.ndim(), paged_v_cache.ndim()); - bool kv_strides_are_identical = true; - for (int i = 0; i < paged_k_cache.ndim(); ++i) { - kv_strides_are_identical &= paged_k_cache.stride(i) == paged_v_cache.stride(i); - } - - ffi::CUDADeviceGuard device_guard(float_workspace_buffer.device().device_id); - const cudaStream_t stream = get_stream(float_workspace_buffer.device()); - - DISPATCH_context( - DTypeQ, DTypeKV, DTypeO, IdType, MASK_MODE, HEAD_DIM_QK, HEAD_DIM_VO, POS_ENCODING_MODE, - USE_SLIDING_WINDOW, USE_LOGITS_SOFT_CAP, USE_FP16_QK_REDUCTION, AttentionVariant, - RaggedParams, PagedParams, [&] { - PagedParams params; - - params.q = static_cast(q.data_ptr()); - paged_kv_t paged_kv( - num_kv_heads, page_size, HEAD_DIM_VO, batch_size, kv_layout, - static_cast(paged_k_cache.data_ptr()), - static_cast(paged_v_cache.data_ptr()), k_cache_strides.data(), - v_cache_strides.data(), static_cast(paged_kv_indices.data_ptr()), - static_cast(paged_kv_indptr.data_ptr()), - static_cast(paged_kv_last_page_len.data_ptr())); - params.paged_kv = paged_kv; - params.q_indptr = static_cast(qo_indptr.data_ptr()); - params.o = static_cast(o.data_ptr()); - - params.lse = maybe_lse ? static_cast(maybe_lse.value().data_ptr()) : nullptr; - params.num_qo_heads = num_qo_heads; - params.group_size = uint_fastdiv(num_qo_heads / paged_kv.num_heads); - params.q_stride_n = q_stride_n; - params.q_stride_h = q_stride_h; - params.window_left = window_left; - - params.request_indices = nullptr; - params.qo_tile_indices = nullptr; - params.kv_tile_indices = nullptr; - params.merge_indptr = nullptr; - params.o_indptr = nullptr; - params.kv_chunk_size_ptr = nullptr; - params.block_valid_mask = nullptr; - params.total_num_rows = nullptr; - params.max_total_num_rows = 0; - params.padded_batch_size = 0; - params.partition_kv = false; - params.k_sf_stride_page = 0; - params.k_sf_stride_n = 0; - params.k_sf_stride_h = 0; - params.v_sf_stride_page = 0; - params.v_sf_stride_n = 0; - params.v_sf_stride_h = 0; - - ADDITIONAL_PARAMS_SETTER - - DTypeO* tmp_v = nullptr; - float* tmp_s = nullptr; - - params.request_indices = - GetPtrFromBaseOffset(int_buffer_ptr, plan_info.request_indices_offset); - params.qo_tile_indices = - GetPtrFromBaseOffset(int_buffer_ptr, plan_info.qo_tile_indices_offset); - params.kv_tile_indices = - GetPtrFromBaseOffset(int_buffer_ptr, plan_info.kv_tile_indices_offset); - params.o_indptr = GetPtrFromBaseOffset(int_buffer_ptr, plan_info.o_indptr_offset); - params.kv_chunk_size_ptr = - GetPtrFromBaseOffset(int_buffer_ptr, plan_info.kv_chunk_size_ptr_offset); - if (plan_info.split_kv) { - params.merge_indptr = - GetPtrFromBaseOffset(int_buffer_ptr, plan_info.merge_indptr_offset); - tmp_v = GetPtrFromBaseOffset(float_buffer_ptr, plan_info.v_offset); - tmp_s = GetPtrFromBaseOffset(float_buffer_ptr, plan_info.s_offset); - if (plan_info.enable_cuda_graph) { - params.block_valid_mask = - GetPtrFromBaseOffset(int_buffer_ptr, plan_info.block_valid_mask_offset); - } - } - params.padded_batch_size = plan_info.padded_batch_size; - params.max_total_num_rows = plan_info.total_num_rows; - if (plan_info.enable_cuda_graph) { - params.total_num_rows = - GetPtrFromBaseOffset(int_buffer_ptr, plan_info.total_num_rows_offset); - } - - cudaError_t status = cudaSuccess; - - DISPATCH_BOOL(kv_strides_are_identical, SAME_KV_STRIDES, [&] { - DISPATCH_CTA_TILE_Q(plan_info.cta_tile_q, CTA_TILE_Q, { - status = flashinfer::BatchPrefillWithPagedKVCacheDispatched< - /*SAME_KV_STRIDES=*/SAME_KV_STRIDES, CTA_TILE_Q, HEAD_DIM_QK, HEAD_DIM_VO, - POS_ENCODING_MODE, /*use_fp16_qk_reduction=*/USE_FP16_QK_REDUCTION, MASK_MODE, - AttentionVariant, PagedParams>(params, tmp_v, tmp_s, enable_pdl, stream); - }); - return true; - }); - - TVM_FFI_ICHECK(status == cudaSuccess) - << "BatchPrefillWithPagedKVCache failed with error " << cudaGetErrorString(status); - return true; - }); -} +#include "batch_prefill_paged.cuh" diff --git a/csrc/batch_prefill_customize_config.jinja b/csrc/batch_prefill_customize_config.jinja index 6f5f1d9aa42..c9ee34e41be 100644 --- a/csrc/batch_prefill_customize_config.jinja +++ b/csrc/batch_prefill_customize_config.jinja @@ -38,6 +38,11 @@ constexpr auto USE_LOGITS_SOFT_CAP = {{ use_logits_soft_cap }}; constexpr auto POS_ENCODING_MODE = {{ pos_encoding_mode }}; constexpr auto USE_SLIDING_WINDOW = {{ use_sliding_window }}; +#define PAGED_KV_STRIDE_MODE_RUNTIME 0 +#define PAGED_KV_STRIDE_MODE_EQUAL 1 +#define PAGED_KV_STRIDE_MODE_INDEPENDENT 2 +#define PAGED_KV_STRIDE_MODE PAGED_KV_STRIDE_MODE_{{ paged_kv_stride_mode | upper }} + struct RaggedParams { using DTypeQ = DTypeQ; diff --git a/csrc/batch_prefill_paged.cu b/csrc/batch_prefill_paged.cu new file mode 100644 index 00000000000..e2e8dd9f1c2 --- /dev/null +++ b/csrc/batch_prefill_paged.cu @@ -0,0 +1,40 @@ +/* + * Copyright (c) 2023 by FlashInfer team. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include +#include +#include + +#include "batch_prefill_config.inc" +#include "tvm/ffi/container/array.h" +#include "tvm_ffi_utils.h" + +namespace flashinfer { + +template +cudaError_t BatchPrefillWithPagedKVCacheDispatched(Params params, typename Params::DTypeO* tmp_v, + float* tmp_s, bool enable_pdl, + cudaStream_t stream); + +} // namespace flashinfer + +using namespace flashinfer; + +using tvm::ffi::Array; +using tvm::ffi::Optional; + +#include "batch_prefill_paged.cuh" diff --git a/csrc/batch_prefill_paged.cuh b/csrc/batch_prefill_paged.cuh new file mode 100644 index 00000000000..66995401dda --- /dev/null +++ b/csrc/batch_prefill_paged.cuh @@ -0,0 +1,181 @@ +/* + * Copyright (c) 2023 by FlashInfer team. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#pragma once + +void BatchPrefillWithPagedKVCacheRun(TensorView float_workspace_buffer, + TensorView int_workspace_buffer, Array plan_info_vec, + TensorView q, TensorView paged_k_cache, + TensorView paged_v_cache, TensorView qo_indptr, + TensorView paged_kv_indptr, TensorView paged_kv_indices, + TensorView paged_kv_last_page_len, TensorView o, + Optional maybe_lse, int64_t mask_mode_code, + int64_t layout, int64_t window_left, + bool enable_pdl ADDITIONAL_FUNC_PARAMS) { + PrefillPlanInfo plan_info; + plan_info.FromVector(std::vector(plan_info_vec.begin(), plan_info_vec.end())); + QKVLayout kv_layout = static_cast(layout); + int64_t batch_size = paged_kv_indptr.size(0) - 1; + int64_t num_qo_heads = q.size(1); + int64_t num_kv_heads, page_size; + uint32_t head_dim_qk = q.size(2); + if (kv_layout == QKVLayout::kHND) { + num_kv_heads = paged_k_cache.size(1); + page_size = paged_k_cache.size(2); + } else { + page_size = paged_k_cache.size(1); + num_kv_heads = paged_k_cache.size(2); + } + + if (maybe_lse) { + const auto& lse = *maybe_lse; + TVM_FFI_ICHECK_EQ(lse.size(0), q.size(0)); + TVM_FFI_ICHECK_EQ(lse.size(1), q.size(1)); + } + + void* float_buffer_ptr = static_cast(float_workspace_buffer.data_ptr()); + void* int_buffer_ptr = static_cast(int_workspace_buffer.data_ptr()); + + const MaskMode mask_mode = static_cast(mask_mode_code); + + // get q_stride_n and q_stride_h + const auto q_stride_n = q.stride(0); + const auto q_stride_h = q.stride(1); + + // get kv-cache strides + auto k_cache_strides = paged_k_cache.strides(); + auto v_cache_strides = paged_v_cache.strides(); + TVM_FFI_ICHECK_EQ(paged_k_cache.ndim(), paged_v_cache.ndim()); + bool kv_strides_are_identical = true; + for (int i = 0; i < paged_k_cache.ndim(); ++i) { + kv_strides_are_identical &= paged_k_cache.stride(i) == paged_v_cache.stride(i); + } + + ffi::CUDADeviceGuard device_guard(float_workspace_buffer.device().device_id); + const cudaStream_t stream = get_stream(float_workspace_buffer.device()); + + DISPATCH_context( + DTypeQ, DTypeKV, DTypeO, IdType, MASK_MODE, HEAD_DIM_QK, HEAD_DIM_VO, POS_ENCODING_MODE, + USE_SLIDING_WINDOW, USE_LOGITS_SOFT_CAP, USE_FP16_QK_REDUCTION, AttentionVariant, + RaggedParams, PagedParams, [&] { + PagedParams params; + + params.q = static_cast(q.data_ptr()); + paged_kv_t paged_kv( + num_kv_heads, page_size, HEAD_DIM_VO, batch_size, kv_layout, + static_cast(paged_k_cache.data_ptr()), + static_cast(paged_v_cache.data_ptr()), k_cache_strides.data(), + v_cache_strides.data(), static_cast(paged_kv_indices.data_ptr()), + static_cast(paged_kv_indptr.data_ptr()), + static_cast(paged_kv_last_page_len.data_ptr())); + params.paged_kv = paged_kv; + params.q_indptr = static_cast(qo_indptr.data_ptr()); + params.o = static_cast(o.data_ptr()); + + params.lse = maybe_lse ? static_cast(maybe_lse.value().data_ptr()) : nullptr; + params.num_qo_heads = num_qo_heads; + params.group_size = uint_fastdiv(num_qo_heads / paged_kv.num_heads); + params.q_stride_n = q_stride_n; + params.q_stride_h = q_stride_h; + params.window_left = window_left; + + params.request_indices = nullptr; + params.qo_tile_indices = nullptr; + params.kv_tile_indices = nullptr; + params.merge_indptr = nullptr; + params.o_indptr = nullptr; + params.kv_chunk_size_ptr = nullptr; + params.block_valid_mask = nullptr; + params.total_num_rows = nullptr; + params.max_total_num_rows = 0; + params.padded_batch_size = 0; + params.partition_kv = false; + params.k_sf_stride_page = 0; + params.k_sf_stride_n = 0; + params.k_sf_stride_h = 0; + params.v_sf_stride_page = 0; + params.v_sf_stride_n = 0; + params.v_sf_stride_h = 0; + + ADDITIONAL_PARAMS_SETTER + + DTypeO* tmp_v = nullptr; + float* tmp_s = nullptr; + + params.request_indices = + GetPtrFromBaseOffset(int_buffer_ptr, plan_info.request_indices_offset); + params.qo_tile_indices = + GetPtrFromBaseOffset(int_buffer_ptr, plan_info.qo_tile_indices_offset); + params.kv_tile_indices = + GetPtrFromBaseOffset(int_buffer_ptr, plan_info.kv_tile_indices_offset); + params.o_indptr = GetPtrFromBaseOffset(int_buffer_ptr, plan_info.o_indptr_offset); + params.kv_chunk_size_ptr = + GetPtrFromBaseOffset(int_buffer_ptr, plan_info.kv_chunk_size_ptr_offset); + if (plan_info.split_kv) { + params.merge_indptr = + GetPtrFromBaseOffset(int_buffer_ptr, plan_info.merge_indptr_offset); + tmp_v = GetPtrFromBaseOffset(float_buffer_ptr, plan_info.v_offset); + tmp_s = GetPtrFromBaseOffset(float_buffer_ptr, plan_info.s_offset); + if (plan_info.enable_cuda_graph) { + params.block_valid_mask = + GetPtrFromBaseOffset(int_buffer_ptr, plan_info.block_valid_mask_offset); + } + } + params.padded_batch_size = plan_info.padded_batch_size; + params.max_total_num_rows = plan_info.total_num_rows; + if (plan_info.enable_cuda_graph) { + params.total_num_rows = + GetPtrFromBaseOffset(int_buffer_ptr, plan_info.total_num_rows_offset); + } + + cudaError_t status = cudaSuccess; + +#if PAGED_KV_STRIDE_MODE == PAGED_KV_STRIDE_MODE_RUNTIME + DISPATCH_BOOL(kv_strides_are_identical, SAME_KV_STRIDES, [&] { + DISPATCH_CTA_TILE_Q(plan_info.cta_tile_q, CTA_TILE_Q, { + status = flashinfer::BatchPrefillWithPagedKVCacheDispatched< + /*SAME_KV_STRIDES=*/SAME_KV_STRIDES, CTA_TILE_Q, HEAD_DIM_QK, HEAD_DIM_VO, + POS_ENCODING_MODE, /*use_fp16_qk_reduction=*/USE_FP16_QK_REDUCTION, MASK_MODE, + AttentionVariant, PagedParams>(params, tmp_v, tmp_s, enable_pdl, stream); + }); + return true; + }); +#elif PAGED_KV_STRIDE_MODE == PAGED_KV_STRIDE_MODE_EQUAL + TVM_FFI_ICHECK(kv_strides_are_identical) + << "The equal-stride FA2 batch-prefill primary requires paged K/V cache data " + "tensors to have identical strides in every dimension; route unequal layouts " + "through the independent paged module."; + DISPATCH_CTA_TILE_Q(plan_info.cta_tile_q, CTA_TILE_Q, { + status = flashinfer::BatchPrefillWithPagedKVCacheDispatched< + /*SAME_KV_STRIDES=*/true, CTA_TILE_Q, HEAD_DIM_QK, HEAD_DIM_VO, POS_ENCODING_MODE, + /*use_fp16_qk_reduction=*/USE_FP16_QK_REDUCTION, MASK_MODE, AttentionVariant, + PagedParams>(params, tmp_v, tmp_s, enable_pdl, stream); + }); +#elif PAGED_KV_STRIDE_MODE == PAGED_KV_STRIDE_MODE_INDEPENDENT + DISPATCH_CTA_TILE_Q(plan_info.cta_tile_q, CTA_TILE_Q, { + status = flashinfer::BatchPrefillWithPagedKVCacheDispatched< + /*SAME_KV_STRIDES=*/false, CTA_TILE_Q, HEAD_DIM_QK, HEAD_DIM_VO, POS_ENCODING_MODE, + /*use_fp16_qk_reduction=*/USE_FP16_QK_REDUCTION, MASK_MODE, AttentionVariant, + PagedParams>(params, tmp_v, tmp_s, enable_pdl, stream); + }); +#else +#error "Unsupported PAGED_KV_STRIDE_MODE" +#endif + + TVM_FFI_ICHECK(status == cudaSuccess) + << "BatchPrefillWithPagedKVCache failed with error " << cudaGetErrorString(status); + return true; + }); +} diff --git a/csrc/batch_prefill_paged_jit_binding.cu b/csrc/batch_prefill_paged_jit_binding.cu new file mode 100644 index 00000000000..e1392547339 --- /dev/null +++ b/csrc/batch_prefill_paged_jit_binding.cu @@ -0,0 +1,32 @@ +/* + * Copyright (c) 2023-2025 by FlashInfer team. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "batch_prefill_config.inc" +#include "tvm_ffi_utils.h" + +using tvm::ffi::Array; +using tvm::ffi::Optional; + +void BatchPrefillWithPagedKVCacheRun(TensorView float_workspace_buffer, + TensorView int_workspace_buffer, Array plan_info_vec, + TensorView q, TensorView paged_k_cache, + TensorView paged_v_cache, TensorView qo_indptr, + TensorView paged_kv_indptr, TensorView paged_kv_indices, + TensorView paged_kv_last_page_len, TensorView o, + Optional maybe_lse, int64_t mask_mode_code, + int64_t layout, int64_t window_left, + bool enable_pdl ADDITIONAL_FUNC_PARAMS); + +TVM_FFI_DLL_EXPORT_TYPED_FUNC(paged_run, BatchPrefillWithPagedKVCacheRun); diff --git a/csrc/batch_prefill_paged_kernel_inst.jinja b/csrc/batch_prefill_paged_kernel_inst.jinja index ff9682c0d2d..93a522b7bd0 100644 --- a/csrc/batch_prefill_paged_kernel_inst.jinja +++ b/csrc/batch_prefill_paged_kernel_inst.jinja @@ -9,7 +9,7 @@ constexpr auto use_custom_mask = {{ mask_mode }} == MaskMode::kCustom; {16} for head_dim_qk >= 512 with a smaller head_dim_vo (o_frag register pressure permits only CTA16 there), and {16, 64, 128} otherwise; don't instantiate unreachable variants. #} -{% for same_kv_strides in ["true", "false"] %} +{% for same_kv_strides in same_kv_strides_values %} {% for cta_tile_q in ([16, 32] if head_dim_vo | int >= 512 else ([16] if head_dim_qk | int >= 512 else [16, 64, 128])) %} template cudaError_t BatchPrefillWithPagedKVCacheDispatched< /*SAME_KV_STRIDES=*/{{same_kv_strides}}, /*CTA_TILE_Q=*/{{cta_tile_q}}, diff --git a/csrc/pod.cu b/csrc/pod.cu index c2a271b0969..24a5ac7fc7e 100644 --- a/csrc/pod.cu +++ b/csrc/pod.cu @@ -123,15 +123,10 @@ void pod_with_kv_cache_tensor( const auto q_stride_n_d = q_d.stride(0); const auto q_stride_h_d = q_d.stride(1); - // get kv_cache_strides - const int64_t* kv_cache_strides_d = nullptr; + // get independent K/V cache strides auto k_strides_d = paged_k_cache_d.strides(); auto v_strides_d = paged_v_cache_d.strides(); TVM_FFI_ICHECK_EQ(k_strides_d.size(), v_strides_d.size()); - for (int i = 0; i < k_strides_d.size(); ++i) { - TVM_FFI_ICHECK_EQ(k_strides_d[i], v_strides_d[i]); - } - kv_cache_strides_d = k_strides_d.data(); ffi::CUDADeviceGuard device_guard(float_workspace_buffer_d.device().device_id); const cudaStream_t stream = get_stream(float_workspace_buffer_d.device()); @@ -187,8 +182,8 @@ void pod_with_kv_cache_tensor( paged_kv_t paged_kv( num_kv_heads, page_size_d, HEAD_DIM_VO, batch_size, kv_layout_d, static_cast(paged_k_cache_d.data_ptr()), - static_cast(paged_v_cache_d.data_ptr()), kv_cache_strides_d, - static_cast(paged_kv_indices_d.data_ptr()), + static_cast(paged_v_cache_d.data_ptr()), k_strides_d.data(), + v_strides_d.data(), static_cast(paged_kv_indices_d.data_ptr()), static_cast(paged_kv_indptr_d.data_ptr()), static_cast(paged_kv_last_page_len_d.data_ptr())); params.paged_kv = paged_kv; diff --git a/flashinfer/aot.py b/flashinfer/aot.py index eeee2e97f74..35b5f3a8f50 100644 --- a/flashinfer/aot.py +++ b/flashinfer/aot.py @@ -44,6 +44,7 @@ gen_trtllm_gen_fmha_module, gen_trtllm_fmha_v2_sm120_module, ) +from .jit.attention.modules import _gen_batch_prefill_primary_module from .jit.attention.utils import _is_nvfp4_kv_dtype from .jit.cascade import gen_cascade_module from .jit.cake_fmha import gen_cake_fmha_compat_module @@ -186,7 +187,7 @@ def gen_fa2( if dtype_qo.itemsize == 1: return # fp8 tensor cores not supported in fa2 - yield gen_batch_prefill_module( + yield _gen_batch_prefill_primary_module( backend="fa2", dtype_q=dtype_qo, dtype_kv=dtype_kv, diff --git a/flashinfer/attention/_core.py b/flashinfer/attention/_core.py index d7d63dddd16..e8bc6117515 100644 --- a/flashinfer/attention/_core.py +++ b/flashinfer/attention/_core.py @@ -390,6 +390,8 @@ def __init__( "use_sliding_window": window_left >= 0, "use_fp16_qk_reduction": use_fp16_qk_reduction, "pos_encoding_mode": PosEncodingMode[pos_encoding_mode].value, + "paged_kv_stride_mode": ("independent" if backend == "fa2" else "runtime"), + "module_surface": "full", } super().__init__( diff --git a/flashinfer/cascade.py b/flashinfer/cascade.py index 67ae92e9f68..a21dc7163e7 100644 --- a/flashinfer/cascade.py +++ b/flashinfer/cascade.py @@ -516,6 +516,12 @@ def plan( kv_data_type=kv_data_type, ) + @flashinfer_api + def prewarm_paged_kv_stride_variant(self, variant: str = "independent") -> None: + r"""Prewarm every level's lazy paged-KV-stride variant after :meth:`plan`.""" + for wrapper in self._batch_prefill_wrappers: + wrapper.prewarm_paged_kv_stride_variant(variant) + begin_forward = plan @flashinfer_api(trace=multi_level_cascade_run_trace) @@ -1002,6 +1008,11 @@ def begin_forward( page_size, ) + @flashinfer_api + def prewarm_paged_kv_stride_variant(self, variant: str = "independent") -> None: + r"""Prewarm the underlying prefill wrapper's paged-KV-stride variant.""" + self._batch_prefill_wrapper.prewarm_paged_kv_stride_variant(variant) + @flashinfer_api def forward( self, diff --git a/flashinfer/decode.py b/flashinfer/decode.py index af3492b0a88..3bf6c6c6ff5 100644 --- a/flashinfer/decode.py +++ b/flashinfer/decode.py @@ -2045,6 +2045,34 @@ def _plan_impl( self._q_len_per_req = q_len_per_req self._is_causal = is_causal + @flashinfer_api + def prewarm_paged_kv_stride_variant(self, variant: str = "independent") -> None: + r"""Load a lazy tensor-core paged-KV-stride variant after :meth:`plan`. + + Call this method before CUDA graph capture when a planned standard FA2 + tensor-core decode wrapper will receive K and V tensors with different + data strides. + """ + if ( + getattr(self, "_plan_info", None) is None + or getattr(self, "_cached_module", None) is None + ): + raise RuntimeError( + "plan() must complete before prewarming a paged-KV-stride variant." + ) + if not self.use_tensor_cores: + raise RuntimeError("Paged-KV-stride prewarm requires tensor-core decode.") + if self._jit_module is not None: + raise RuntimeError( + "Paged-KV-stride prewarm is not supported for a custom JIT module." + ) + if self._backend != "fa2": + raise RuntimeError( + "Paged-KV-stride prewarm requires a standard FA2 plan, " + f"got backend={self._backend!r}." + ) + self._cached_module.prewarm_paged_kv_stride_variant(variant) + begin_forward = plan def forward( diff --git a/flashinfer/jit/attention/modules.py b/flashinfer/jit/attention/modules.py index 9571d0cfdf1..bfd73350dc1 100755 --- a/flashinfer/jit/attention/modules.py +++ b/flashinfer/jit/attention/modules.py @@ -15,7 +15,7 @@ """ import os -from typing import Any, List, Optional +from typing import Any, List, Literal, Optional import jinja2 import torch @@ -104,6 +104,62 @@ def plan(self, *args: object) -> object: return self.plan_with_staged_workspace_bytes(*args)[0] +BatchPrefillPagedKVStrideMode = Literal["runtime", "equal", "independent"] +BatchPrefillModuleSurface = Literal["full", "paged"] + +_BATCH_PREFILL_MODULE_URI_SUFFIX = { + ("runtime", "full"): "", + ("equal", "full"): "_kv_stride_equal", + ("independent", "full"): "_kv_stride_independent", + ("independent", "paged"): "_paged_kv_stride_independent", +} + + +def _validate_batch_prefill_module_mode( + backend: str, + paged_kv_stride_mode: BatchPrefillPagedKVStrideMode, + module_surface: BatchPrefillModuleSurface, +) -> None: + if paged_kv_stride_mode not in ("runtime", "equal", "independent"): + raise ValueError( + "paged_kv_stride_mode must be one of 'runtime', 'equal', or " + f"'independent', got {paged_kv_stride_mode!r}" + ) + if module_surface not in ("full", "paged"): + raise ValueError( + f"module_surface must be either 'full' or 'paged', got {module_surface!r}" + ) + if (paged_kv_stride_mode, module_surface) not in _BATCH_PREFILL_MODULE_URI_SUFFIX: + raise ValueError( + "Unsupported batch-prefill mode/surface combination: " + f"mode={paged_kv_stride_mode!r}, surface={module_surface!r}" + ) + if backend != "fa2" and (paged_kv_stride_mode, module_surface) != ( + "runtime", + "full", + ): + raise ValueError( + "Paged KV stride specialization is supported only for the FA2 " + "backend, got " + f"backend={backend!r}, mode={paged_kv_stride_mode!r}, " + f"surface={module_surface!r}" + ) + + +def _get_batch_prefill_module_uri( + base_uri: str, + backend: str, + paged_kv_stride_mode: BatchPrefillPagedKVStrideMode, + module_surface: BatchPrefillModuleSurface, +) -> str: + """Return a content-unique URI for one validated batch-prefill artifact.""" + _validate_batch_prefill_module_mode(backend, paged_kv_stride_mode, module_surface) + return ( + base_uri + + _BATCH_PREFILL_MODULE_URI_SUFFIX[(paged_kv_stride_mode, module_surface)] + ) + + def get_single_decode_uri( dtype_q: torch.dtype, dtype_kv: torch.dtype, @@ -1022,7 +1078,7 @@ def gen_batch_decode_module( ) -def gen_batch_prefill_module( +def _gen_batch_prefill_module( backend: str, dtype_q: torch.dtype, dtype_kv: torch.dtype, @@ -1034,8 +1090,11 @@ def gen_batch_prefill_module( use_sliding_window: bool, use_logits_soft_cap: bool, use_fp16_qk_reduction: bool, + *, + paged_kv_stride_mode: BatchPrefillPagedKVStrideMode, + module_surface: BatchPrefillModuleSurface, ) -> JitSpec: - uri = get_batch_prefill_uri( + base_uri = get_batch_prefill_uri( backend, dtype_q, dtype_kv, @@ -1048,6 +1107,9 @@ def gen_batch_prefill_module( use_logits_soft_cap, use_fp16_qk_reduction, ) + uri = _get_batch_prefill_module_uri( + base_uri, backend, paged_kv_stride_mode, module_surface + ) # use `fp8_enabled` flag to use separate kernel template # this is used for fp8 tensor core computation @@ -1148,6 +1210,132 @@ def gen_batch_prefill_module( use_logits_soft_cap=use_logits_soft_cap, use_fp16_qk_reduction=use_fp16_qk_reduction, fp8_enabled=fp8_enabled, + paged_kv_stride_mode=paged_kv_stride_mode, + module_surface=module_surface, + ) + + +def gen_batch_prefill_module( + backend: str, + dtype_q: torch.dtype, + dtype_kv: torch.dtype, + dtype_o: torch.dtype, + dtype_idx: torch.dtype, + head_dim_qk: int, + head_dim_vo: int, + pos_encoding_mode: int, + use_sliding_window: bool, + use_logits_soft_cap: bool, + use_fp16_qk_reduction: bool, +) -> JitSpec: + """Generate the public full batch-prefill module with runtime stride dispatch.""" + return _gen_batch_prefill_module( + backend, + dtype_q, + dtype_kv, + dtype_o, + dtype_idx, + head_dim_qk, + head_dim_vo, + pos_encoding_mode, + use_sliding_window, + use_logits_soft_cap, + use_fp16_qk_reduction, + paged_kv_stride_mode="runtime", + module_surface="full", + ) + + +def _gen_batch_prefill_primary_module( + backend: str, + dtype_q: torch.dtype, + dtype_kv: torch.dtype, + dtype_o: torch.dtype, + dtype_idx: torch.dtype, + head_dim_qk: int, + head_dim_vo: int, + pos_encoding_mode: int, + use_sliding_window: bool, + use_logits_soft_cap: bool, + use_fp16_qk_reduction: bool, +) -> JitSpec: + """Generate the internal full FA2 primary with equal-stride paged kernels.""" + return _gen_batch_prefill_module( + backend, + dtype_q, + dtype_kv, + dtype_o, + dtype_idx, + head_dim_qk, + head_dim_vo, + pos_encoding_mode, + use_sliding_window, + use_logits_soft_cap, + use_fp16_qk_reduction, + paged_kv_stride_mode="equal", + module_surface="full", + ) + + +def _gen_batch_prefill_independent_full_module( + backend: str, + dtype_q: torch.dtype, + dtype_kv: torch.dtype, + dtype_o: torch.dtype, + dtype_idx: torch.dtype, + head_dim_qk: int, + head_dim_vo: int, + pos_encoding_mode: int, + use_sliding_window: bool, + use_logits_soft_cap: bool, + use_fp16_qk_reduction: bool, +) -> JitSpec: + """Generate the feasibility-only full FA2 independent-stride module.""" + return _gen_batch_prefill_module( + backend, + dtype_q, + dtype_kv, + dtype_o, + dtype_idx, + head_dim_qk, + head_dim_vo, + pos_encoding_mode, + use_sliding_window, + use_logits_soft_cap, + use_fp16_qk_reduction, + paged_kv_stride_mode="independent", + module_surface="full", + ) + + +def _gen_batch_prefill_independent_paged_module( + backend: str, + dtype_q: torch.dtype, + dtype_kv: torch.dtype, + dtype_o: torch.dtype, + dtype_idx: torch.dtype, + head_dim_qk: int, + head_dim_vo: int, + pos_encoding_mode: int, + use_sliding_window: bool, + use_logits_soft_cap: bool, + use_fp16_qk_reduction: bool, +) -> JitSpec: + """Generate the internal paged-only FA2 independent-stride module.""" + return _gen_batch_prefill_module( + backend, + dtype_q, + dtype_kv, + dtype_o, + dtype_idx, + head_dim_qk, + head_dim_vo, + pos_encoding_mode, + use_sliding_window, + use_logits_soft_cap, + use_fp16_qk_reduction, + paged_kv_stride_mode="independent", + module_surface="paged", ) @@ -1196,6 +1384,8 @@ def gen_batch_prefill_attention_sink_module( use_logits_soft_cap=False, use_fp16_qk_reduction=False, fp8_enabled=False, + paged_kv_stride_mode="independent" if backend == "fa2" else "runtime", + module_surface="full", ) @@ -1679,7 +1869,10 @@ def gen_customize_batch_prefill_module( use_logits_soft_cap: bool = False, use_fp16_qk_reduction: bool = False, fp8_enabled: bool = False, + paged_kv_stride_mode: BatchPrefillPagedKVStrideMode = "runtime", + module_surface: BatchPrefillModuleSurface = "full", ) -> JitSpec: + _validate_batch_prefill_module_mode(backend, paged_kv_stride_mode, module_surface) require_fp4_kv_cache = dtype_map_kv[dtype_kv] == "__nv_fp4x2_e2m1" if require_fp4_kv_cache: missing_sf_tensors = [ @@ -1708,6 +1901,12 @@ def gen_customize_batch_prefill_module( "use_sliding_window": str(use_sliding_window).lower(), "use_logits_soft_cap": str(use_logits_soft_cap).lower(), "use_fp16_qk_reduction": str(use_fp16_qk_reduction).lower(), + "paged_kv_stride_mode": paged_kv_stride_mode, + "same_kv_strides_values": { + "runtime": ["true", "false"], + "equal": ["true"], + "independent": ["false"], + }[paged_kv_stride_mode], } if backend == "auto": raise ValueError("backend should not be auto when jit_args is provided") @@ -1735,10 +1934,12 @@ def gen_customize_batch_prefill_module( ) as f: paged_kernel_inst_templ = jinja2.Template(f.read()) - with open( - jit_env.FLASHINFER_CSRC_DIR / "batch_prefill_ragged_kernel_inst.jinja" - ) as f: - ragged_kernel_inst_templ = jinja2.Template(f.read()) + ragged_kernel_inst_templ = None + if module_surface == "full": + with open( + jit_env.FLASHINFER_CSRC_DIR / "batch_prefill_ragged_kernel_inst.jinja" + ) as f: + ragged_kernel_inst_templ = jinja2.Template(f.read()) kwargs |= { "additional_params_decl": additional_params_decl, @@ -1763,20 +1964,26 @@ def gen_customize_batch_prefill_module( ) write_if_different(dest_path, source) - dest_path = ( - gen_directory / f"batch_prefill_ragged_kernel_mask_{mask_mode}.cu" - ) - source_paths.append(dest_path) - source = ragged_kernel_inst_templ.render( - mask_mode=mask_mode_literal[mask_mode], - **kwargs, - ) - write_if_different(dest_path, source) - - for filename in [ - "batch_prefill.cu", - "batch_prefill_jit_binding.cu", - ]: + if ragged_kernel_inst_templ is not None: + dest_path = ( + gen_directory / f"batch_prefill_ragged_kernel_mask_{mask_mode}.cu" + ) + source_paths.append(dest_path) + source = ragged_kernel_inst_templ.render( + mask_mode=mask_mode_literal[mask_mode], + **kwargs, + ) + write_if_different(dest_path, source) + + host_sources = ( + ["batch_prefill.cu", "batch_prefill_jit_binding.cu"] + if module_surface == "full" + else [ + "batch_prefill_paged.cu", + "batch_prefill_paged_jit_binding.cu", + ] + ) + for filename in host_sources: src_path = jit_env.FLASHINFER_CSRC_DIR / filename dest_path = gen_directory / filename source_paths.append(dest_path) @@ -1784,6 +1991,11 @@ def gen_customize_batch_prefill_module( source = f.read() write_if_different(dest_path, source) + paged_header = "batch_prefill_paged.cuh" + with open(jit_env.FLASHINFER_CSRC_DIR / paged_header, "r") as f: + source = f.read() + write_if_different(gen_directory / paged_header, source) + generated_config_path = gen_directory / "batch_prefill_config.inc" write_if_different(generated_config_path, generated_inc_str) extra_cuda_cflags = _fa2_prefill_head_dim_nvcc_flags( diff --git a/flashinfer/prefill.py b/flashinfer/prefill.py index ccf192e9ca8..0501d27480d 100755 --- a/flashinfer/prefill.py +++ b/flashinfer/prefill.py @@ -17,6 +17,7 @@ import functools import logging import math +import threading from types import SimpleNamespace from typing import Any, Dict, List, Literal, Optional, Tuple, Union, overload @@ -25,6 +26,7 @@ from .api_logging import flashinfer_api from .cudnn import cudnn_batch_prefill_with_kv_cache from .jit import ( + MissingJITCacheError, gen_batch_prefill_module, gen_customize_batch_prefill_module, gen_fmha_cutlass_sm100a_module, @@ -36,6 +38,10 @@ get_single_prefill_uri, setup_cubin_loader, ) +from .jit.attention.modules import ( + _gen_batch_prefill_independent_paged_module, + _gen_batch_prefill_primary_module, +) from .jit.attention.utils import _is_nvfp4_kv_dtype from .page import get_seq_lens from .quantization import packbits, segment_packbits @@ -455,8 +461,65 @@ def _fake_run_single_prefill( return SimpleNamespace(run=run_single_prefill) +class _LazyBatchPrefillIndependentModule: + """Load one independent-stride batch-prefill module on first eager use.""" + + def __init__(self, spec: Any) -> None: + self._spec = spec + self._lock = threading.Lock() + self._module: Optional[Any] = None + + @property + def is_loaded(self) -> bool: + """Whether this holder already has an in-process loaded module.""" + return self._module is not None + + @staticmethod + def _check_not_capturing() -> None: + if torch.cuda.is_current_stream_capturing(): + raise RuntimeError( + "The lazy independent paged-KV-stride module cannot be compiled " + "or loaded during CUDA graph capture. Call " + "prewarm_paged_kv_stride_variant('independent') after plan() " + "and before capture." + ) + + def get(self) -> Any: + """Return the loaded module, building it once outside graph capture.""" + module = self._module + if module is not None: + return module + + self._check_not_capturing() + with self._lock: + module = self._module + if module is not None: + return module + self._check_not_capturing() + try: + module = self._spec.build_and_load() + except MissingJITCacheError as exc: + raise MissingJITCacheError( + "Unequal K/V data strides require FlashInfer's lazy " + "independent paged module, which is not included in the " + "default JIT cache. Use equal-stride K/V tensors, enable " + "local JIT and call " + "prewarm_paged_kv_stride_variant('independent') after " + "plan(), or install a compatible independent-module cache " + "package when one becomes available.", + spec=exc.spec, + ) from exc + self._module = module + return module + + def prewarm(self) -> None: + """Eagerly load the independent module for later graph capture.""" + self.get() + + @functools.cache def get_batch_prefill_module(backend, *args): + lazy_independent_module: Optional[_LazyBatchPrefillIndependentModule] = None if backend == "trtllm-gen": uri = "trtllm_gen_context" module = get_trtllm_gen_prefill_module() @@ -464,6 +527,16 @@ def get_batch_prefill_module(backend, *args): workspace_size_func = None ragged_run_func = module.ragged_run paged_run_func = module.paged_run + elif backend == "fa2": + uri = get_batch_prefill_uri(backend, *args) + module = _gen_batch_prefill_primary_module(backend, *args).build_and_load() + lazy_independent_module = _LazyBatchPrefillIndependentModule( + _gen_batch_prefill_independent_paged_module(backend, *args) + ) + plan_func = module.plan + workspace_size_func = getattr(module, "workspace_size", None) + ragged_run_func = module.ragged_run + paged_run_func = module.paged_run else: uri = get_batch_prefill_uri(backend, *args) module = gen_batch_prefill_module(backend, *args).build_and_load() @@ -748,7 +821,12 @@ def paged_run( ) elif backend == "fa2": assert not is_float8(q) - paged_run_func( + if tuple(paged_k_cache.stride()) == tuple(paged_v_cache.stride()): + routed_paged_run_func = paged_run_func + else: + assert lazy_independent_module is not None + routed_paged_run_func = lazy_independent_module.get().paged_run + routed_paged_run_func( float_workspace_buffer, int_workspace_buffer, plan_info_vec, @@ -896,11 +974,23 @@ def _fake_paged_run( # # Note that plan is not part of model logic. It should not be included in # Cuda Graph or torch.compile. So, we don't provide a torch library for plan. + def prewarm_paged_kv_stride_variant(variant: str = "independent") -> None: + """Load the requested lazy paged-KV-stride variant before graph capture.""" + if variant != "independent": + raise ValueError(f"variant must be 'independent', got {variant!r}") + if lazy_independent_module is None: + raise RuntimeError( + "Paged-KV-stride prewarm is available only for standard FA2 " + f"batch-prefill modules, got backend={backend!r}." + ) + lazy_independent_module.prewarm() + return SimpleNamespace( plan=plan_func, workspace_size=workspace_size_func, ragged_run=ragged_run, paged_run=paged_run, + prewarm_paged_kv_stride_variant=prewarm_paged_kv_stride_variant, ) @@ -2754,6 +2844,32 @@ def plan( self._seq_lens_kv = seq_lens self._seq_lens_q = seq_lens_q if seq_lens_q is not None else seq_lens + @flashinfer_api + def prewarm_paged_kv_stride_variant(self, variant: str = "independent") -> None: + r"""Load a lazy paged-KV-stride variant after :meth:`plan`. + + Call this method before CUDA graph capture when a planned standard FA2 + wrapper will receive K and V tensors with different data strides. + """ + if ( + getattr(self, "_plan_info", None) is None + or getattr(self, "_cached_module", None) is None + ): + raise RuntimeError( + "plan() must complete before prewarming a paged-KV-stride variant." + ) + if self._jit_module is not None: + raise RuntimeError( + "Paged-KV-stride prewarm is not supported for a custom JIT module." + ) + if self._backend != "fa2": + raise RuntimeError( + "Paged-KV-stride prewarm requires a standard FA2 plan, " + f"got backend={self._backend!r}." + ) + assert self._cached_module is not None + self._cached_module.prewarm_paged_kv_stride_variant(variant) + begin_forward = plan def forward( diff --git a/tests/attention/test_batch_prefill_kernels.py b/tests/attention/test_batch_prefill_kernels.py index 7eb441668a4..48dafc5212f 100644 --- a/tests/attention/test_batch_prefill_kernels.py +++ b/tests/attention/test_batch_prefill_kernels.py @@ -20,6 +20,10 @@ import pytest import torch from tests.test_helpers.jit_utils import gen_prefill_attention_modules +from tests.test_helpers.paged_kv import ( + get_closure_value, + make_padded_paged_kv_view, +) import flashinfer from tests.test_helpers.test_helpers import assert_close_chunked, ref_single_prefill @@ -401,6 +405,385 @@ def test_batch_prefill_equal_kv_strides_bf16(): torch.testing.assert_close(actual.float(), expected.float(), rtol=1e-2, atol=1e-2) +@pytest.mark.parametrize("kv_layout", ["NHD", "HND"]) +@pytest.mark.parametrize("disable_split_kv", [False, True]) +@pytest.mark.parametrize("head_dim", [64, 128]) +def test_batch_prefill_lazy_stride_router_plan_reuse( + kv_layout, disable_split_kv, head_dim +): + """Reuse one equal-primary plan across equal/unequal/equal paged runs.""" + torch.manual_seed(42) + batch_size, qo_len, kv_len, page_size = 2, 17, 97, 16 + num_qo_heads, num_kv_heads = 8, 2 + pages_per_request = (kv_len + page_size - 1) // page_size + total_pages = batch_size * pages_per_request + + q = torch.randn( + batch_size * qo_len, + num_qo_heads, + head_dim, + device="cuda", + dtype=torch.bfloat16, + ) + if kv_layout == "NHD": + cache_shape = (total_pages, page_size, num_kv_heads, head_dim) + to_dense = lambda cache: cache.reshape(-1, num_kv_heads, head_dim) + else: + cache_shape = (total_pages, num_kv_heads, page_size, head_dim) + to_dense = lambda cache: cache.permute(0, 2, 1, 3).reshape( + -1, num_kv_heads, head_dim + ) + + k = torch.randn(cache_shape, device="cuda", dtype=torch.bfloat16) / 4 + v_equal = torch.randn(cache_shape, device="cuda", dtype=torch.bfloat16) / 4 + v_unequal = make_padded_paged_kv_view(v_equal, kv_layout) + assert k.shape == v_equal.shape == v_unequal.shape + assert k.stride() == v_equal.stride() + assert k.stride() != v_unequal.stride() + + qo_indptr = ( + torch.arange(0, batch_size + 1, device="cuda", dtype=torch.int32) * qo_len + ) + kv_indptr = ( + torch.arange(0, batch_size + 1, device="cuda", dtype=torch.int32) + * pages_per_request + ) + kv_indices = torch.arange(total_pages, device="cuda", dtype=torch.int32) + last_page_len = torch.full( + (batch_size,), + (kv_len - 1) % page_size + 1, + device="cuda", + dtype=torch.int32, + ) + + wrapper = flashinfer.BatchPrefillWithPagedKVCacheWrapper( + torch.empty(128 * 1024 * 1024, device="cuda", dtype=torch.uint8), + kv_layout, + backend="fa2", + ) + wrapper.plan( + qo_indptr, + kv_indptr, + kv_indices, + last_page_len, + num_qo_heads, + num_kv_heads, + head_dim, + page_size, + causal=True, + q_data_type=torch.bfloat16, + kv_data_type=torch.bfloat16, + fixed_split_size=None if disable_split_kv else 64, + disable_split_kv=disable_split_kv, + ) + plan_info = tuple(wrapper._plan_info) + outputs = [ + wrapper.run(q, cache) for cache in ((k, v_equal), (k, v_unequal), (k, v_equal)) + ] + assert tuple(wrapper._plan_info) == plan_info + + expected_batches = [] + group_size = num_qo_heads // num_kv_heads + for batch_idx in range(batch_size): + q_i = q[batch_idx * qo_len : (batch_idx + 1) * qo_len].float() + page_slice = slice( + batch_idx * pages_per_request, + (batch_idx + 1) * pages_per_request, + ) + k_i = to_dense(k[page_slice])[:kv_len].repeat_interleave(group_size, dim=1) + v_i = to_dense(v_equal[page_slice])[:kv_len].repeat_interleave( + group_size, dim=1 + ) + logits = torch.einsum("qhd,khd->hqk", q_i, k_i.float()) * head_dim**-0.5 + q_pos = torch.arange(qo_len, device="cuda").unsqueeze(1) + k_pos = torch.arange(kv_len, device="cuda").unsqueeze(0) + logits.masked_fill_( + ~(k_pos <= q_pos + (kv_len - qo_len)).unsqueeze(0), float("-inf") + ) + expected_batches.append( + torch.einsum("hqk,khd->qhd", torch.softmax(logits, dim=-1), v_i.float()) + ) + expected = torch.cat(expected_batches) + for output in outputs: + torch.testing.assert_close(output.float(), expected, rtol=2e-2, atol=2e-2) + torch.testing.assert_close(outputs[0], outputs[1], rtol=1e-2, atol=1e-2) + torch.testing.assert_close(outputs[0], outputs[2], rtol=1e-2, atol=1e-2) + + +def test_batch_prefill_lazy_stride_router_nvfp4(monkeypatch): + """Route on data strides while preserving independent NVFP4 scale strides.""" + torch.manual_seed(42) + batch_size, qo_len, kv_len, page_size = 1, 17, 33, 16 + num_qo_heads, num_kv_heads, head_dim = 4, 2, 128 + pages_per_request = (kv_len + page_size - 1) // page_size + total_pages = batch_size * pages_per_request + + q = torch.randn( + batch_size * qo_len, + num_qo_heads, + head_dim, + device="cuda", + dtype=torch.float16, + ) + packed_shape = (total_pages, page_size, num_kv_heads, head_dim // 2) + k_packed, k_sf, k_scale = create_nvfp4_kv(packed_shape, "cuda") + v_packed, v_sf, v_scale = create_nvfp4_kv(packed_shape, "cuda") + v_packed_unequal = make_padded_paged_kv_view(v_packed, "NHD") + k_sf_unequal = make_padded_paged_kv_view(k_sf, "NHD", padding_heads=1) + v_sf_unequal = make_padded_paged_kv_view(v_sf, "NHD", padding_heads=3) + assert k_packed.stride() == v_packed.stride() + assert k_packed.stride() != v_packed_unequal.stride() + assert k_sf_unequal.stride() != v_sf_unequal.stride() + + qo_indptr = torch.tensor([0, qo_len], device="cuda", dtype=torch.int32) + kv_indptr = torch.tensor([0, pages_per_request], device="cuda", dtype=torch.int32) + kv_indices = torch.arange(total_pages, device="cuda", dtype=torch.int32) + last_page_len = torch.tensor( + [(kv_len - 1) % page_size + 1], device="cuda", dtype=torch.int32 + ) + custom_mask = torch.tril( + torch.ones(qo_len, kv_len, device="cuda", dtype=torch.bool), + diagonal=kv_len - qo_len, + ).flatten() + wrapper = flashinfer.BatchPrefillWithPagedKVCacheWrapper( + torch.empty(128 * 1024 * 1024, device="cuda", dtype=torch.uint8), + "NHD", + backend="fa2", + ) + wrapper.plan( + qo_indptr, + kv_indptr, + kv_indices, + last_page_len, + num_qo_heads, + num_kv_heads, + head_dim, + page_size, + custom_mask=custom_mask, + q_data_type=torch.float16, + kv_data_type=torch.uint8, + ) + plan_info = tuple(wrapper._plan_info) + holder = get_closure_value( + wrapper._cached_module.paged_run, "lazy_independent_module" + ) + + def unexpected_supplement_route(): + raise AssertionError("scale stride asymmetry must not select the supplement") + + with monkeypatch.context() as patch: + patch.setattr(holder, "get", unexpected_supplement_route) + equal_output = wrapper.run( + q, + (k_packed, v_packed), + kv_cache_sf=(k_sf_unequal, v_sf_unequal), + k_scale=k_scale.item(), + v_scale=v_scale.item(), + ) + + wrapper.prewarm_paged_kv_stride_variant("independent") + unequal_output = wrapper.run( + q, + (k_packed, v_packed_unequal), + kv_cache_sf=(k_sf_unequal, v_sf_unequal), + k_scale=k_scale.item(), + v_scale=v_scale.item(), + ) + assert holder.is_loaded + assert tuple(wrapper._plan_info) == plan_info + torch.testing.assert_close(unequal_output, equal_output, rtol=1e-3, atol=1e-3) + + +def test_batch_prefill_lazy_stride_router_ragged_primary(monkeypatch): + """Keep ragged execution on the full equal-primary module.""" + torch.manual_seed(42) + batch_size, qo_len, kv_len = 2, 7, 11 + num_qo_heads, num_kv_heads, head_dim = 4, 2, 64 + q = torch.randn( + batch_size * qo_len, + num_qo_heads, + head_dim, + device="cuda", + dtype=torch.float16, + ) + k = torch.randn( + batch_size * kv_len, + num_kv_heads, + head_dim, + device="cuda", + dtype=torch.float16, + ) + v = torch.randn_like(k) + qo_indptr = torch.arange(batch_size + 1, device="cuda", dtype=torch.int32) * qo_len + kv_indptr = torch.arange(batch_size + 1, device="cuda", dtype=torch.int32) * kv_len + wrapper = flashinfer.BatchPrefillWithRaggedKVCacheWrapper( + torch.empty(128 * 1024 * 1024, device="cuda", dtype=torch.uint8), + "NHD", + backend="fa2", + ) + wrapper.plan( + qo_indptr, + kv_indptr, + num_qo_heads, + num_kv_heads, + head_dim, + causal=True, + q_data_type=torch.float16, + kv_data_type=torch.float16, + ) + holder = get_closure_value( + wrapper._cached_module.paged_run, "lazy_independent_module" + ) + + def unexpected_supplement_route(): + raise AssertionError("ragged execution must remain on the primary module") + + monkeypatch.setattr(holder, "get", unexpected_supplement_route) + output = wrapper.run(q, k, v) + for batch_idx in range(batch_size): + query_slice = slice(batch_idx * qo_len, (batch_idx + 1) * qo_len) + kv_slice = slice(batch_idx * kv_len, (batch_idx + 1) * kv_len) + expected = flashinfer.single_prefill_with_kv_cache( + q[query_slice], k[kv_slice], v[kv_slice], causal=True, backend="fa2" + ) + torch.testing.assert_close(output[query_slice], expected, rtol=1e-3, atol=1e-3) + + +def _make_lazy_stride_router_graph_case(): + batch_size, qo_len, kv_len, page_size = 2, 17, 97, 16 + num_qo_heads, num_kv_heads, head_dim = 8, 2, 128 + pages_per_request = (kv_len + page_size - 1) // page_size + total_pages = batch_size * pages_per_request + q = torch.randn( + batch_size * qo_len, + num_qo_heads, + head_dim, + device="cuda", + dtype=torch.bfloat16, + ) + k = torch.randn( + total_pages, + page_size, + num_kv_heads, + head_dim, + device="cuda", + dtype=torch.bfloat16, + ) + v_equal = torch.randn_like(k) + v_unequal = make_padded_paged_kv_view(v_equal, "NHD") + qo_indptr = torch.arange(batch_size + 1, device="cuda", dtype=torch.int32) * qo_len + kv_indptr = ( + torch.arange(batch_size + 1, device="cuda", dtype=torch.int32) + * pages_per_request + ) + kv_indices = torch.arange(total_pages, device="cuda", dtype=torch.int32) + last_page_len = torch.full( + (batch_size,), + (kv_len - 1) % page_size + 1, + device="cuda", + dtype=torch.int32, + ) + wrapper = flashinfer.BatchPrefillWithPagedKVCacheWrapper( + torch.empty(128 * 1024 * 1024, device="cuda", dtype=torch.uint8), + "NHD", + backend="fa2", + ) + wrapper.plan( + qo_indptr, + kv_indptr, + kv_indices, + last_page_len, + num_qo_heads, + num_kv_heads, + head_dim, + page_size, + causal=True, + q_data_type=torch.bfloat16, + kv_data_type=torch.bfloat16, + fixed_split_size=64, + ) + return wrapper, q, k, v_equal, v_unequal + + +def test_batch_prefill_lazy_stride_router_cuda_graph(): + """A post-plan prewarm makes the unequal route capture/replay safe.""" + torch.manual_seed(42) + wrapper, q, k, v_equal, v_unequal = _make_lazy_stride_router_graph_case() + plan_info = tuple(wrapper._plan_info) + equal_output = wrapper.run(q, (k, v_equal), enable_pdl=False) + wrapper.prewarm_paged_kv_stride_variant("independent") + + graph_output = torch.empty_like(equal_output) + warmup_stream = torch.cuda.Stream() + warmup_stream.wait_stream(torch.cuda.current_stream()) + with torch.cuda.stream(warmup_stream): + for _ in range(3): + wrapper.run( + q, + (k, v_unequal), + out=graph_output, + enable_pdl=False, + ) + torch.cuda.current_stream().wait_stream(warmup_stream) + graph = torch.cuda.CUDAGraph() + with torch.cuda.graph(graph): + wrapper.run( + q, + (k, v_unequal), + out=graph_output, + enable_pdl=False, + ) + graph_output.fill_(float("nan")) + graph.replay() + torch.cuda.synchronize() + assert torch.isfinite(graph_output).all() + assert tuple(wrapper._plan_info) == plan_info + torch.testing.assert_close(graph_output, equal_output, rtol=1e-2, atol=1e-2) + + +def test_batch_prefill_lazy_stride_router_torch_compile(): + """Dynamo guards both stride families without rebuilding on equal return.""" + torch.manual_seed(42) + wrapper, q, k, v_equal, v_unequal = _make_lazy_stride_router_graph_case() + wrapper.prewarm_paged_kv_stride_variant("independent") + expected = wrapper.run(q, (k, v_equal), enable_pdl=False) + torch._dynamo.reset() + from torch._dynamo.utils import guard_failures + + guard_failures.clear() + compiled_graphs = [] + + def counting_backend(graph_module, _example_inputs): + compiled_graphs.append(graph_module) + return graph_module.forward + + def run_for_compile(q_arg, k_arg, v_arg, out_arg): + return wrapper.run( + q_arg, + (k_arg, v_arg), + out=out_arg, + enable_pdl=False, + ).clone() + + compiled_run = torch.compile( + run_for_compile, + backend=counting_backend, + fullgraph=False, + dynamic=False, + ) + graph_counts = [] + for v_cache in (v_equal, v_equal, v_unequal, v_unequal, v_equal): + actual = compiled_run(q, k, v_cache, torch.empty_like(expected)) + torch.testing.assert_close(actual, expected, rtol=1e-2, atol=1e-2) + graph_counts.append(len(compiled_graphs)) + failures = [str(reason) for reason in guard_failures[run_for_compile.__code__]] + assert len(failures) == 1 and "stride mismatch" in failures[0] + assert len(compiled_graphs) == 1 + assert graph_counts[1] == graph_counts[0] + assert graph_counts[3] == graph_counts[2] + assert graph_counts[4] == graph_counts[3] + + @pytest.mark.parametrize("causal", [False, True]) @pytest.mark.parametrize("pos_encoding_mode", ["NONE", "ROPE_LLAMA"]) def test_batch_prefill_with_paged_kv_cache_head_dim_512( diff --git a/tests/attention/test_block_sparse.py b/tests/attention/test_block_sparse.py index 83e0d02b867..7b66ba26b2f 100644 --- a/tests/attention/test_block_sparse.py +++ b/tests/attention/test_block_sparse.py @@ -30,6 +30,7 @@ gen_decode_attention_modules, gen_prefill_attention_modules, ) +from tests.test_helpers.paged_kv import make_padded_view import flashinfer from flashinfer.cutile.cutile_common import is_cuda_tile_available @@ -237,6 +238,50 @@ def test_block_sparse_attention( ) +def test_block_sparse_lazy_stride_router(): + """Run one tensor-core block-sparse plan with equal and unequal KV strides.""" + set_seed(42) + M = N = 32 + R = C = 16 + num_qo_heads, num_kv_heads, head_dim = 4, 2, 128 + indptr = torch.tensor([0, 2, 4], dtype=torch.int32, device="cuda") + indices = torch.tensor([0, 1, 0, 1], dtype=torch.int32, device="cuda") + q = torch.randn(M, num_qo_heads, head_dim, dtype=torch.float16, device="cuda") + k = torch.randn(N, num_kv_heads, head_dim, dtype=torch.float16, device="cuda") + v_equal = torch.randn_like(k) + v_unequal = make_padded_view(v_equal, 1) + assert k.stride() == v_equal.stride() + assert k.stride() != v_unequal.stride() + + wrapper = flashinfer.sparse.BlockSparseAttentionWrapper( + torch.empty(128 * 1024 * 1024, dtype=torch.uint8, device="cuda"), + backend="fa2", + ) + wrapper.plan( + indptr, + indices, + M, + N, + R, + C, + num_qo_heads, + num_kv_heads, + head_dim, + q_data_type=torch.float16, + kv_data_type=torch.float16, + ) + plan_info = tuple(wrapper._plan_info) + routed_module = wrapper._cached_module + equal_output = wrapper.run(q, k, v_equal) + unequal_output = wrapper.run(q, k, v_unequal) + assert wrapper._cached_module is routed_module + assert tuple(wrapper._plan_info) == plan_info + + expected = flashinfer.single_prefill_with_kv_cache(q, k, v_equal, backend="fa2") + torch.testing.assert_close(equal_output, expected, atol=1e-2, rtol=1e-3) + torch.testing.assert_close(unequal_output, expected, atol=1e-2, rtol=1e-3) + + def _ref_attention( q: torch.Tensor, # [gqa_group_size, qo_len, head_dim] k: torch.Tensor, # [1, kv_len, head_dim] diff --git a/tests/attention/test_shared_prefix_kernels.py b/tests/attention/test_shared_prefix_kernels.py index 30aee0dc388..717be73f260 100644 --- a/tests/attention/test_shared_prefix_kernels.py +++ b/tests/attention/test_shared_prefix_kernels.py @@ -20,6 +20,7 @@ gen_decode_attention_modules, gen_prefill_attention_modules, ) +from tests.test_helpers.paged_kv import make_padded_paged_kv_view import flashinfer from flashinfer.utils import has_flashinfer_jit_cache @@ -230,6 +231,106 @@ def test_batch_attention_with_shared_prefix_paged_kv_cache( torch.testing.assert_close(o_multi_level, o_two_level, rtol=1e-3, atol=1e-3) +def test_shared_prefix_and_cascade_lazy_stride_router(): + """Share one routed FA2 namespace across cascade and shared-prefix prefill.""" + torch.manual_seed(42) + batch_size, qo_len, shared_kv_len, unique_kv_len = 2, 8, 16, 16 + page_size = 16 + num_qo_heads, num_kv_heads, head_dim = 4, 2, 128 + q = torch.randn( + batch_size * qo_len, + num_qo_heads, + head_dim, + device="cuda", + dtype=torch.float16, + ) + k_cache = torch.randn( + 1 + batch_size, + page_size, + num_kv_heads, + head_dim, + device="cuda", + dtype=torch.float16, + ) + v_cache_equal = torch.randn_like(k_cache) + v_cache_unequal = make_padded_paged_kv_view(v_cache_equal, "NHD") + k_shared = k_cache[0, :shared_kv_len] + v_shared = v_cache_equal[0, :shared_kv_len] + + qo_indptr_top = torch.tensor( + [0, batch_size * qo_len], dtype=torch.int32, device="cuda" + ) + qo_indptr_bottom = ( + torch.arange(batch_size + 1, dtype=torch.int32, device="cuda") * qo_len + ) + shared_kv_indptr = torch.tensor([0, 1], dtype=torch.int32, device="cuda") + shared_kv_indices = torch.tensor([0], dtype=torch.int32, device="cuda") + shared_last_page_len = torch.tensor( + [shared_kv_len], dtype=torch.int32, device="cuda" + ) + unique_kv_indptr = torch.arange(batch_size + 1, dtype=torch.int32, device="cuda") + unique_kv_indices = torch.arange( + 1, 1 + batch_size, dtype=torch.int32, device="cuda" + ) + unique_last_page_len = torch.full( + (batch_size,), unique_kv_len, dtype=torch.int32, device="cuda" + ) + workspace = torch.empty(128 * 1024 * 1024, dtype=torch.uint8, device="cuda") + cascade = flashinfer.MultiLevelCascadeAttentionWrapper(2, workspace, "NHD") + cascade.plan( + [qo_indptr_top, qo_indptr_bottom], + [shared_kv_indptr, unique_kv_indptr], + [shared_kv_indices, unique_kv_indices], + [shared_last_page_len, unique_last_page_len], + num_qo_heads, + num_kv_heads, + head_dim, + page_size, + causal=False, + ) + shared_prefix = flashinfer.BatchPrefillWithSharedPrefixPagedKVCacheWrapper( + workspace, "NHD" + ) + shared_prefix.begin_forward( + qo_indptr_bottom, + unique_kv_indptr, + unique_kv_indices, + unique_last_page_len, + num_qo_heads, + num_kv_heads, + head_dim, + page_size, + ) + modules = [wrapper._cached_module for wrapper in cascade._batch_prefill_wrappers] + assert modules[0] is modules[1] + assert shared_prefix._batch_prefill_wrapper._cached_module is modules[0] + cascade_plans = [ + tuple(wrapper._plan_info) for wrapper in cascade._batch_prefill_wrappers + ] + shared_prefix_plan = tuple(shared_prefix._batch_prefill_wrapper._plan_info) + + equal_cache = (k_cache, v_cache_equal) + unequal_cache = (k_cache, v_cache_unequal) + cascade_equal = cascade.run(q, equal_cache) + prefix_equal = shared_prefix.forward( + q, k_shared, v_shared, equal_cache, causal=False + ) + cascade.prewarm_paged_kv_stride_variant("independent") + shared_prefix.prewarm_paged_kv_stride_variant("independent") + cascade_unequal = cascade.run(q, unequal_cache) + prefix_unequal = shared_prefix.forward( + q, k_shared, v_shared, unequal_cache, causal=False + ) + + assert [ + tuple(wrapper._plan_info) for wrapper in cascade._batch_prefill_wrappers + ] == cascade_plans + assert tuple(shared_prefix._batch_prefill_wrapper._plan_info) == shared_prefix_plan + torch.testing.assert_close(cascade_equal, prefix_equal, rtol=1e-3, atol=1e-3) + torch.testing.assert_close(cascade_unequal, prefix_equal, rtol=1e-3, atol=1e-3) + torch.testing.assert_close(prefix_unequal, prefix_equal, rtol=1e-3, atol=1e-3) + + @pytest.mark.parametrize("seed", [0]) @pytest.mark.parametrize("num_tries", [50]) def test_merge_state_in_place_with_mask(seed, num_tries): diff --git a/tests/attention/test_tensor_cores_decode.py b/tests/attention/test_tensor_cores_decode.py index 39e57c14c13..2047fe6e3c0 100644 --- a/tests/attention/test_tensor_cores_decode.py +++ b/tests/attention/test_tensor_cores_decode.py @@ -20,6 +20,7 @@ gen_decode_attention_modules, gen_prefill_attention_modules, ) +from tests.test_helpers.paged_kv import make_padded_paged_kv_view from functools import partial import flashinfer from flashinfer.utils import has_flashinfer_jit_cache @@ -264,6 +265,115 @@ def test_batch_decode_tensor_cores_equal_kv_strides_bf16_group_size_5(): torch.testing.assert_close(actual.float(), expected, rtol=1e-2, atol=1e-2) +def test_batch_decode_lazy_stride_router(): + """Prewarm and capture the independently strided tensor-core decode path.""" + torch.manual_seed(42) + batch_size, kv_len, page_size = 2, 97, 16 + num_qo_heads, num_kv_heads, head_dim = 40, 8, 128 + pages_per_request = (kv_len + page_size - 1) // page_size + total_pages = batch_size * pages_per_request + + q = torch.randn( + batch_size, + num_qo_heads, + head_dim, + device="cuda", + dtype=torch.bfloat16, + ) + k = torch.randn( + total_pages, + page_size, + num_kv_heads, + head_dim, + device="cuda", + dtype=torch.bfloat16, + ) + v_equal = torch.randn_like(k) + v_unequal = make_padded_paged_kv_view(v_equal, "NHD") + assert k.stride() == v_equal.stride() + assert k.stride() != v_unequal.stride() + + kv_indptr = ( + torch.arange(batch_size + 1, device="cuda", dtype=torch.int32) + * pages_per_request + ) + kv_indices = torch.arange(total_pages, device="cuda", dtype=torch.int32) + last_page_len = torch.full( + (batch_size,), + (kv_len - 1) % page_size + 1, + device="cuda", + dtype=torch.int32, + ) + workspace = torch.empty(128 * 1024 * 1024, device="cuda", dtype=torch.uint8) + wrapper = flashinfer.BatchDecodeWithPagedKVCacheWrapper( + workspace, + "NHD", + use_cuda_graph=True, + use_tensor_cores=True, + paged_kv_indptr_buffer=torch.empty_like(kv_indptr), + paged_kv_indices_buffer=torch.empty_like(kv_indices), + paged_kv_last_page_len_buffer=torch.empty_like(last_page_len), + backend="fa2", + ) + wrapper.plan( + kv_indptr, + kv_indices, + last_page_len, + num_qo_heads, + num_kv_heads, + head_dim, + page_size, + q_data_type=torch.bfloat16, + kv_data_type=torch.bfloat16, + ) + plan_info = tuple(wrapper._plan_info) + equal_output = wrapper.run(q, (k, v_equal), enable_pdl=False).clone() + wrapper.prewarm_paged_kv_stride_variant("independent") + unequal_output = wrapper.run(q, (k, v_unequal), enable_pdl=False).clone() + + graph_output = torch.empty_like(equal_output) + warmup_stream = torch.cuda.Stream() + warmup_stream.wait_stream(torch.cuda.current_stream()) + with torch.cuda.stream(warmup_stream): + for _ in range(3): + wrapper.run( + q, + (k, v_unequal), + out=graph_output, + enable_pdl=False, + ) + torch.cuda.current_stream().wait_stream(warmup_stream) + + graph = torch.cuda.CUDAGraph() + with torch.cuda.graph(graph): + wrapper.run( + q, + (k, v_unequal), + out=graph_output, + enable_pdl=False, + ) + graph_output.fill_(float("nan")) + graph.replay() + torch.cuda.synchronize() + assert tuple(wrapper._plan_info) == plan_info + + k_dense = k.view(batch_size, pages_per_request * page_size, num_kv_heads, head_dim)[ + :, :kv_len + ] + v_dense = v_equal.view( + batch_size, pages_per_request * page_size, num_kv_heads, head_dim + )[:, :kv_len] + group_size = num_qo_heads // num_kv_heads + k_dense = k_dense.repeat_interleave(group_size, dim=2).float() + v_dense = v_dense.repeat_interleave(group_size, dim=2).float() + logits = torch.einsum("bhd,bkhd->bhk", q.float(), k_dense) * head_dim**-0.5 + expected = torch.einsum("bhk,bkhd->bhd", torch.softmax(logits, dim=-1), v_dense) + + torch.testing.assert_close(equal_output.float(), expected, rtol=1e-2, atol=1e-2) + torch.testing.assert_close(unequal_output.float(), expected, rtol=1e-2, atol=1e-2) + torch.testing.assert_close(graph_output.float(), expected, rtol=1e-2, atol=1e-2) + + @pytest.mark.parametrize("batch_size", [12, 17]) @pytest.mark.parametrize("kv_len", [54, 97, 512]) @pytest.mark.parametrize("page_size", [1, 8, 16]) diff --git a/tests/jit/test_attention_utils.py b/tests/jit/test_attention_utils.py index 62f31e7ece3..dbcd4f25b66 100644 --- a/tests/jit/test_attention_utils.py +++ b/tests/jit/test_attention_utils.py @@ -1,5 +1,7 @@ +import inspect from pathlib import Path +import pytest import torch from flashinfer.jit import core @@ -12,6 +14,12 @@ def test_batch_prefill_nvfp4_swa_paged_params_declares_sf_strides( tmp_path, monkeypatch ): repo_root = Path(__file__).resolve().parents[2] + monkeypatch.setattr(core, "check_cuda_arch", lambda: None) + monkeypatch.setattr( + attention_modules.current_compilation_context, + "TARGET_CUDA_ARCHS", + {(10, 3)}, + ) monkeypatch.setattr(jit_env, "FLASHINFER_GEN_SRC_DIR", tmp_path / "generated") monkeypatch.setattr(jit_env, "FLASHINFER_CSRC_DIR", repo_root / "csrc") @@ -70,45 +78,292 @@ def test_batch_prefill_generates_equal_and_independent_stride_specializations( monkeypatch.setattr(jit_env, "FLASHINFER_GEN_SRC_DIR", tmp_path / "generated") monkeypatch.setattr(jit_env, "FLASHINFER_CSRC_DIR", repo_root / "csrc") - uri = "test_batch_prefill_stride_specializations" - spec = gen_customize_batch_prefill_module( + public_parameters = inspect.signature( + attention_modules.gen_batch_prefill_module + ).parameters + assert list(public_parameters) == [ + "backend", + "dtype_q", + "dtype_kv", + "dtype_o", + "dtype_idx", + "head_dim_qk", + "head_dim_vo", + "pos_encoding_mode", + "use_sliding_window", + "use_logits_soft_cap", + "use_fp16_qk_reduction", + ] + assert all( + parameter.default is inspect.Parameter.empty + and parameter.kind is inspect.Parameter.POSITIONAL_OR_KEYWORD + for parameter in public_parameters.values() + ) + + args = ( "fa2", - uri, torch.bfloat16, torch.bfloat16, torch.bfloat16, torch.int32, 64, 64, - [], - [], - [], - [], - "DefaultAttention", - "struct DefaultAttention {};", + 0, + False, + False, + False, ) + base_uri = attention_modules.get_batch_prefill_uri(*args) + specs = { + "runtime_full": attention_modules.gen_batch_prefill_module(*args), + "equal_full": attention_modules._gen_batch_prefill_primary_module(*args), + "independent_full": ( + attention_modules._gen_batch_prefill_independent_full_module(*args) + ), + "independent_paged": ( + attention_modules._gen_batch_prefill_independent_paged_module(*args) + ), + } - assert spec.name == uri - paged = tmp_path / "generated" / uri / "batch_prefill_paged_kernel_mask_0.cu" - ragged = tmp_path / "generated" / uri / "batch_prefill_ragged_kernel_mask_0.cu" - binding = tmp_path / "generated" / uri / "batch_prefill.cu" - paged_text = paged.read_text() - assert ( - paged_text.count("template cudaError_t BatchPrefillWithPagedKVCacheDispatched<") - == 6 + assert {variant: spec.name for variant, spec in specs.items()} == { + "runtime_full": base_uri, + "equal_full": f"{base_uri}_kv_stride_equal", + "independent_full": f"{base_uri}_kv_stride_independent", + "independent_paged": f"{base_uri}_paged_kv_stride_independent", + } + assert len({spec.name for spec in specs.values()}) == 4 + + full_sources = sorted( + [f"batch_prefill_paged_kernel_mask_{mask_mode}.cu" for mask_mode in range(4)] + + [f"batch_prefill_ragged_kernel_mask_{mask_mode}.cu" for mask_mode in range(4)] + + ["batch_prefill.cu", "batch_prefill_jit_binding.cu"] ) - assert paged_text.count("/*SAME_KV_STRIDES=*/true") == 3 - assert paged_text.count("/*SAME_KV_STRIDES=*/false") == 3 - assert ( - ragged.read_text().count( - "template cudaError_t BatchPrefillWithRaggedKVCacheDispatched<" + paged_sources = sorted( + [f"batch_prefill_paged_kernel_mask_{mask_mode}.cu" for mask_mode in range(4)] + + ["batch_prefill_paged.cu", "batch_prefill_paged_jit_binding.cu"] + ) + expected_sources = { + "runtime_full": full_sources, + "equal_full": full_sources, + "independent_full": full_sources, + "independent_paged": paged_sources, + } + expected_instantiations = { + "runtime_full": {"true": 3, "false": 3}, + "equal_full": {"true": 3, "false": 0}, + "independent_full": {"true": 0, "false": 3}, + "independent_paged": {"true": 0, "false": 3}, + } + expected_mode_macros = { + "runtime_full": "PAGED_KV_STRIDE_MODE_RUNTIME", + "equal_full": "PAGED_KV_STRIDE_MODE_EQUAL", + "independent_full": "PAGED_KV_STRIDE_MODE_INDEPENDENT", + "independent_paged": "PAGED_KV_STRIDE_MODE_INDEPENDENT", + } + + for variant, spec in specs.items(): + generated_dir = tmp_path / "generated" / spec.name + assert {source.parent for source in spec.sources} == {generated_dir} + assert ( + sorted(source.name for source in spec.sources) == expected_sources[variant] ) - == 3 + assert len(spec.sources) == (6 if variant == "independent_paged" else 10) + + config = (generated_dir / "batch_prefill_config.inc").read_text() + assert "#define PAGED_KV_STRIDE_MODE " + expected_mode_macros[variant] in config + paged_header = generated_dir / "batch_prefill_paged.cuh" + assert paged_header.exists() + assert paged_header not in spec.sources + paged_host_impl = paged_header.read_text() + assert "paged_k_cache.ndim()" in paged_host_impl + assert "route unequal layouts" in paged_host_impl + assert "through the independent paged module" in paged_host_impl + + is_paged_only = variant == "independent_paged" + host_name = "batch_prefill_paged.cu" if is_paged_only else "batch_prefill.cu" + binding_name = ( + "batch_prefill_paged_jit_binding.cu" + if is_paged_only + else "batch_prefill_jit_binding.cu" + ) + host = (generated_dir / host_name).read_text() + assert '#include "batch_prefill_paged.cuh"' in host + binding = (generated_dir / binding_name).read_text() + assert "TVM_FFI_DLL_EXPORT_TYPED_FUNC(paged_run" in binding + assert binding.count("TVM_FFI_DLL_EXPORT_TYPED_FUNC(") == ( + 1 if is_paged_only else 4 + ) + if is_paged_only: + for excluded_export in ("plan", "workspace_size", "ragged_run"): + assert f"TVM_FFI_DLL_EXPORT_TYPED_FUNC({excluded_export}" not in binding + + for mask_mode in range(4): + paged_text = ( + generated_dir / f"batch_prefill_paged_kernel_mask_{mask_mode}.cu" + ).read_text() + assert paged_text.count( + "template cudaError_t BatchPrefillWithPagedKVCacheDispatched<" + ) == sum(expected_instantiations[variant].values()) + for value, count in expected_instantiations[variant].items(): + assert paged_text.count(f"/*SAME_KV_STRIDES=*/{value}") == count + + ragged_path = ( + generated_dir / f"batch_prefill_ragged_kernel_mask_{mask_mode}.cu" + ) + if is_paged_only: + assert not ragged_path.exists() + else: + assert ( + ragged_path.read_text().count( + "template cudaError_t BatchPrefillWithRaggedKVCacheDispatched<" + ) + == 3 + ) + + for invalid_mode in ("runtime", "equal"): + invalid_uri = f"invalid_{invalid_mode}_paged_surface" + with pytest.raises( + ValueError, match="Unsupported batch-prefill mode/surface combination" + ): + gen_customize_batch_prefill_module( + "fa2", + invalid_uri, + torch.bfloat16, + torch.bfloat16, + torch.bfloat16, + torch.int32, + 64, + 64, + [], + [], + [], + [], + "DefaultAttention", + "struct DefaultAttention {};", + paged_kv_stride_mode=invalid_mode, + module_surface="paged", + ) + assert not (tmp_path / "generated" / invalid_uri).exists() + + +def test_gen_fa2_aot_uses_equal_primary_only(monkeypatch): + import flashinfer.aot as flashinfer_aot + + sentinel = object() + calls = [] + + def fake_primary(**kwargs): + calls.append(kwargs) + return sentinel + + def unexpected_public_full(**kwargs): + raise AssertionError("gen_fa2 must not put the runtime/full module in AOT") + + monkeypatch.setattr( + flashinfer_aot, "_gen_batch_prefill_primary_module", fake_primary ) - assert ( - "DISPATCH_BOOL(kv_strides_are_identical, SAME_KV_STRIDES" in binding.read_text() + monkeypatch.setattr( + flashinfer_aot, "gen_batch_prefill_module", unexpected_public_full ) + specs = list( + flashinfer_aot.gen_fa2( + dtype_qo=torch.bfloat16, + dtype_kv=torch.bfloat16, + head_dim_qk=128, + head_dim_vo=128, + use_sliding_window=False, + use_logits_soft_cap=False, + prefill_only=True, + ) + ) + + assert specs == [sentinel] + assert len(calls) == 1 + assert calls[0]["backend"] == "fa2" + + +def test_gen_attention_aot_matrix_contains_equal_primaries_only(tmp_path, monkeypatch): + import flashinfer.aot as flashinfer_aot + + repo_root = Path(__file__).resolve().parents[2] + monkeypatch.setattr(core, "check_cuda_arch", lambda: None) + monkeypatch.setattr( + attention_modules.current_compilation_context, + "TARGET_CUDA_ARCHS", + {(10, 3)}, + ) + monkeypatch.setattr(jit_env, "FLASHINFER_GEN_SRC_DIR", tmp_path / "generated") + monkeypatch.setattr(jit_env, "FLASHINFER_CSRC_DIR", repo_root / "csrc") + + specs = list( + flashinfer_aot.gen_attention( + f16_dtype_=[torch.bfloat16], + f8_dtype_=[], + fa2_head_dim_=[(64, 64), (128, 128)], + fa3_head_dim_=[], + use_sliding_window_=[False, True], + use_logits_soft_cap_=[False], + has_sm90=False, + has_sm100=False, + add_gemma=False, + add_oai_oss=False, + ) + ) + standard_prefill_specs = [ + spec for spec in specs if spec.name.startswith("batch_prefill_with_kv_cache_") + ] + + assert len(standard_prefill_specs) == 4 + assert all( + spec.name.endswith("_kv_stride_equal") for spec in standard_prefill_specs + ) + assert all(len(spec.sources) == 10 for spec in standard_prefill_specs) + assert not any( + "_paged_kv_stride_independent" in spec.name + or spec.name.endswith("_kv_stride_independent") + for spec in specs + ) + + +def test_attention_sink_stays_independent_full(tmp_path, monkeypatch): + repo_root = Path(__file__).resolve().parents[2] + monkeypatch.setattr(core, "check_cuda_arch", lambda: None) + monkeypatch.setattr( + attention_modules.current_compilation_context, + "TARGET_CUDA_ARCHS", + {(10, 3)}, + ) + monkeypatch.setattr(jit_env, "FLASHINFER_GEN_SRC_DIR", tmp_path / "generated") + monkeypatch.setattr(jit_env, "FLASHINFER_CSRC_DIR", repo_root / "csrc") + + args = ( + "fa2", + torch.bfloat16, + torch.bfloat16, + torch.bfloat16, + torch.int32, + 64, + 64, + 0, + False, + ) + spec = attention_modules.gen_batch_prefill_attention_sink_module(*args) + assert spec.name == attention_modules.get_batch_prefill_attention_sink_uri(*args) + assert len(spec.sources) == 10 + + generated_dir = tmp_path / "generated" / spec.name + config = (generated_dir / "batch_prefill_config.inc").read_text() + assert "#define PAGED_KV_STRIDE_MODE PAGED_KV_STRIDE_MODE_INDEPENDENT" in config + binding = (generated_dir / "batch_prefill_jit_binding.cu").read_text() + assert binding.count("TVM_FFI_DLL_EXPORT_TYPED_FUNC(") == 4 + for mask_mode in range(4): + paged_text = ( + generated_dir / f"batch_prefill_paged_kernel_mask_{mask_mode}.cu" + ).read_text() + assert paged_text.count("/*SAME_KV_STRIDES=*/true") == 0 + assert paged_text.count("/*SAME_KV_STRIDES=*/false") == 3 + def test_batch_prefill_nvfp4_requires_sf_tensors(): try: diff --git a/tests/jit/test_batch_prefill_lazy_loader.py b/tests/jit/test_batch_prefill_lazy_loader.py new file mode 100644 index 00000000000..771de8a8b8a --- /dev/null +++ b/tests/jit/test_batch_prefill_lazy_loader.py @@ -0,0 +1,309 @@ +from concurrent.futures import ThreadPoolExecutor +from types import SimpleNamespace +import threading +import time + +import pytest +import torch + +import flashinfer.prefill as prefill +from flashinfer.cascade import ( + BatchPrefillWithSharedPrefixPagedKVCacheWrapper, + MultiLevelCascadeAttentionWrapper, +) +from flashinfer.decode import BatchDecodeWithPagedKVCacheWrapper +from flashinfer.jit import MissingJITCacheError + + +_MODULE_ARGS = ( + torch.bfloat16, + torch.bfloat16, + torch.bfloat16, + torch.int32, + 128, + 128, + 0, + False, + False, + False, +) + + +class _FakeSpec: + def __init__(self, module, *, error=None, delay_seconds=0.0): + self.module = module + self.error = error + self.delay_seconds = delay_seconds + self.build_count = 0 + self._counter_lock = threading.Lock() + + def build_and_load(self): + with self._counter_lock: + self.build_count += 1 + if self.delay_seconds: + time.sleep(self.delay_seconds) + if self.error is not None: + raise self.error + return self.module + + +def _raw_module(label, calls): + def paged_run(*args): + calls.append(label) + + return SimpleNamespace( + plan=lambda *args: [1], + workspace_size=lambda *args: 0, + ragged_run=lambda *args: None, + paged_run=paged_run, + ) + + +def _run_paged(module, paged_k_cache, paged_v_cache): + tensor = torch.empty(1) + return module.paged_run( + tensor, + tensor, + [1], + tensor, + paged_k_cache, + paged_v_cache, + tensor, + tensor, + tensor, + tensor, + tensor, + None, + 0, + 0, + -1, + False, + None, + None, + None, + None, + None, + None, + 0.0, + 1.0, + None, + None, + None, + 1.0, + 1.0, + 0, + 0, + ) + + +@pytest.fixture(autouse=True) +def _clear_batch_prefill_module_cache(): + prefill.get_batch_prefill_module.cache_clear() + yield + prefill.get_batch_prefill_module.cache_clear() + + +def test_lazy_holder_capture_order_warm_fast_path_and_no_jit(monkeypatch): + loaded_module = object() + spec = _FakeSpec(loaded_module) + holder = prefill._LazyBatchPrefillIndependentModule(spec) + capture_states = iter((False, True)) + monkeypatch.setattr( + torch.cuda, "is_current_stream_capturing", lambda: next(capture_states) + ) + + with pytest.raises(RuntimeError, match="before capture"): + holder.get() + assert spec.build_count == 0 + assert not holder.is_loaded + + monkeypatch.setattr(torch.cuda, "is_current_stream_capturing", lambda: False) + holder.prewarm() + assert holder.is_loaded + assert spec.build_count == 1 + + def unexpected_capture_check(): + raise AssertionError("warm fast path must not query capture state") + + monkeypatch.setattr( + torch.cuda, "is_current_stream_capturing", unexpected_capture_check + ) + assert holder.get() is loaded_module + assert spec.build_count == 1 + + missing_spec = object() + no_jit_spec = _FakeSpec( + object(), + error=MissingJITCacheError("generic cache miss", spec=missing_spec), + ) + no_jit_holder = prefill._LazyBatchPrefillIndependentModule(no_jit_spec) + monkeypatch.setattr(torch.cuda, "is_current_stream_capturing", lambda: False) + with pytest.raises(MissingJITCacheError) as exc_info: + no_jit_holder.get() + assert exc_info.value.spec is missing_spec + assert isinstance(exc_info.value.__cause__, MissingJITCacheError) + message = str(exc_info.value) + assert "Unequal K/V data strides" in message + assert "not included in the default JIT cache" in message + assert "prewarm_paged_kv_stride_variant('independent')" in message + + +def test_concurrent_first_use_loads_once(monkeypatch): + loaded_module = object() + spec = _FakeSpec(loaded_module, delay_seconds=0.05) + holder = prefill._LazyBatchPrefillIndependentModule(spec) + monkeypatch.setattr(torch.cuda, "is_current_stream_capturing", lambda: False) + + with ThreadPoolExecutor(max_workers=8) as pool: + modules = list(pool.map(lambda _: holder.get(), range(8))) + + assert all(module is loaded_module for module in modules) + assert spec.build_count == 1 + + +def test_fa2_routes_complete_data_strides_under_one_logical_operation(monkeypatch): + calls = [] + primary_spec = _FakeSpec(_raw_module("primary", calls)) + independent_spec = _FakeSpec(_raw_module("independent", calls)) + registered_ops = [] + + def record_custom_op(name, *args, **kwargs): + def decorator(func): + registered_ops.append(name) + return func + + return decorator + + monkeypatch.setattr(prefill, "register_custom_op", record_custom_op) + monkeypatch.setattr( + prefill, "register_fake_op", lambda *args, **kwargs: lambda f: f + ) + monkeypatch.setattr( + prefill, "_gen_batch_prefill_primary_module", lambda *args: primary_spec + ) + monkeypatch.setattr( + prefill, + "_gen_batch_prefill_independent_paged_module", + lambda *args: independent_spec, + ) + monkeypatch.setattr(torch.cuda, "is_current_stream_capturing", lambda: False) + + module = prefill.get_batch_prefill_module("fa2", *_MODULE_ARGS) + logical_uri = prefill.get_batch_prefill_uri("fa2", *_MODULE_ARGS) + assert registered_ops == [ + f"flashinfer::{logical_uri}_ragged_run", + f"flashinfer::{logical_uri}_paged_run", + ] + assert primary_spec.build_count == 1 + assert independent_spec.build_count == 0 + + shape = (2, 3, 4, 5) + equal_k = torch.empty_strided(shape, (60, 20, 5, 1)) + equal_v = torch.empty_strided(shape, (60, 20, 5, 1)) + unequal_v = torch.empty_strided(shape, (80, 20, 5, 1)) + _run_paged(module, equal_k, equal_v) + _run_paged(module, equal_k, unequal_v) + _run_paged(module, equal_k, equal_v) + + assert calls == ["primary", "independent", "primary"] + assert independent_spec.build_count == 1 + module.prewarm_paged_kv_stride_variant("independent") + assert independent_spec.build_count == 1 + with pytest.raises(ValueError, match="variant must be 'independent'"): + module.prewarm_paged_kv_stride_variant("unknown") + + +def test_non_fa2_keeps_public_full_module_and_rejects_prewarm(monkeypatch): + calls = [] + public_spec = _FakeSpec(_raw_module("public", calls)) + monkeypatch.setattr(prefill, "gen_batch_prefill_module", lambda *args: public_spec) + + def unexpected_specialized_generator(*args): + raise AssertionError("non-FA2 must not construct a stride-specialized module") + + monkeypatch.setattr( + prefill, "_gen_batch_prefill_primary_module", unexpected_specialized_generator + ) + monkeypatch.setattr( + prefill, + "_gen_batch_prefill_independent_paged_module", + unexpected_specialized_generator, + ) + + module = prefill.get_batch_prefill_module("fa3", *_MODULE_ARGS) + shape = (2, 3, 4, 5) + _run_paged( + module, + torch.empty_strided(shape, (60, 20, 5, 1)), + torch.empty_strided(shape, (80, 20, 5, 1)), + ) + assert calls == ["public"] + assert public_spec.build_count == 1 + with pytest.raises(RuntimeError, match="only for standard FA2"): + module.prewarm_paged_kv_stride_variant() + + +class _PrewarmRecorder: + def __init__(self): + self.variants = [] + + def prewarm_paged_kv_stride_variant(self, variant): + self.variants.append(variant) + + +def test_standard_wrappers_expose_post_plan_prewarm(): + cached_modules = [] + + prefill_wrapper = prefill.BatchPrefillWithPagedKVCacheWrapper.__new__( + prefill.BatchPrefillWithPagedKVCacheWrapper + ) + prefill_wrapper._plan_info = [1] + prefill_wrapper._cached_module = _PrewarmRecorder() + prefill_wrapper._jit_module = None + prefill_wrapper._backend = "fa2" + cached_modules.append(prefill_wrapper._cached_module) + + decode_wrapper = BatchDecodeWithPagedKVCacheWrapper.__new__( + BatchDecodeWithPagedKVCacheWrapper + ) + decode_wrapper._plan_info = [1] + decode_wrapper._cached_module = _PrewarmRecorder() + decode_wrapper._jit_module = None + decode_wrapper._backend = "fa2" + decode_wrapper._use_tensor_cores = True + cached_modules.append(decode_wrapper._cached_module) + + for wrapper in (prefill_wrapper, decode_wrapper): + wrapper.prewarm_paged_kv_stride_variant("independent") + + assert all(module.variants == ["independent"] for module in cached_modules) + + unplanned = prefill.BatchPrefillWithPagedKVCacheWrapper.__new__( + prefill.BatchPrefillWithPagedKVCacheWrapper + ) + unplanned._plan_info = None + unplanned._cached_module = None + unplanned._jit_module = None + unplanned._backend = "fa2" + with pytest.raises(RuntimeError, match=r"plan\(\) must complete"): + unplanned.prewarm_paged_kv_stride_variant() + + +def test_cascade_wrappers_delegate_prewarm(): + level_one = _PrewarmRecorder() + level_two = _PrewarmRecorder() + multi_level = MultiLevelCascadeAttentionWrapper.__new__( + MultiLevelCascadeAttentionWrapper + ) + multi_level._batch_prefill_wrappers = [level_one, level_two] + multi_level.prewarm_paged_kv_stride_variant("independent") + assert level_one.variants == ["independent"] + assert level_two.variants == ["independent"] + + prefill_child = _PrewarmRecorder() + shared_prefill = BatchPrefillWithSharedPrefixPagedKVCacheWrapper.__new__( + BatchPrefillWithSharedPrefixPagedKVCacheWrapper + ) + shared_prefill._batch_prefill_wrapper = prefill_child + shared_prefill.prewarm_paged_kv_stride_variant("independent") + assert prefill_child.variants == ["independent"] diff --git a/tests/test_helpers/paged_kv.py b/tests/test_helpers/paged_kv.py new file mode 100644 index 00000000000..91998980233 --- /dev/null +++ b/tests/test_helpers/paged_kv.py @@ -0,0 +1,73 @@ +# Copyright (c) 2026 by FlashInfer team. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Small helpers for constructing deliberate paged-KV stride families.""" + +from __future__ import annotations + +from collections.abc import Callable +from typing import Any + +import torch + + +def make_padded_view( + source: torch.Tensor, + dimension: int, + *, + padding: int = 2, +) -> torch.Tensor: + """Copy ``source`` into an equal-shaped view with padding in one dimension.""" + if padding <= 0: + raise ValueError(f"padding must be positive, got {padding}") + dimension %= source.ndim + parent_shape = list(source.shape) + parent_shape[dimension] += padding + parent = torch.empty( + parent_shape, + dtype=source.dtype, + device=source.device, + ) + view = parent.narrow(dimension, 0, source.shape[dimension]) + view.copy_(source) + if view.stride() == source.stride(): + raise AssertionError( + f"padding dimension {dimension} did not change strides for {source.shape}" + ) + return view + + +def make_padded_paged_kv_view( + source: torch.Tensor, + kv_layout: str, + *, + padding_heads: int = 2, +) -> torch.Tensor: + """Return an equal-shaped paged-KV view with a padded head dimension.""" + if source.ndim != 4: + raise ValueError(f"expected rank-4 paged KV, got shape {source.shape}") + if kv_layout == "NHD": + head_dimension = 2 + elif kv_layout == "HND": + head_dimension = 1 + else: + raise ValueError(f"kv_layout must be 'NHD' or 'HND', got {kv_layout!r}") + return make_padded_view(source, head_dimension, padding=padding_heads) + + +def get_closure_value(function: Callable[..., Any], name: str) -> Any: + """Return a named closure value from a routed test-facing callable.""" + closure = function.__closure__ or () + values = dict(zip(function.__code__.co_freevars, closure, strict=True)) + return values[name].cell_contents diff --git a/tests/trace/test_batch_pod_run_reference_correctness.py b/tests/trace/test_batch_pod_run_reference_correctness.py index 9f30d49f731..268de165d37 100644 --- a/tests/trace/test_batch_pod_run_reference_correctness.py +++ b/tests/trace/test_batch_pod_run_reference_correctness.py @@ -6,6 +6,7 @@ from tests.trace.reference_utils import ( _check, ) +from tests.test_helpers.paged_kv import make_padded_paged_kv_view @pytest.mark.parametrize( @@ -86,3 +87,90 @@ def test_batch_pod_run_reference_correctness(shape_kwargs): _check(batch_pod_with_paged_kv_cache_run_trace, ref_d, out_d, atol=1e-3, rtol=1e-3) if torch.cuda.is_available(): torch.cuda.synchronize() + + +def test_batch_pod_independent_paged_kv_strides(): + """Reuse one fused BatchPOD module across unequal prefill and decode KV.""" + from flashinfer.pod import BatchPODWithPagedKVCacheWrapper, get_batch_pod_module + from flashinfer.trace.templates.attention import ( + batch_pod_with_paged_kv_cache_run_trace, + ) + + inputs = batch_pod_with_paged_kv_cache_run_trace.init( + device="cuda", + prefill_len=8, + decode_batch_size=1, + num_pages=2, + num_qo_heads=8, + num_kv_heads=2, + head_dim=64, + page_size=8, + ) + plan = inputs["plan"] + run = inputs["run"] + k_cache_p, v_cache_p = run["paged_kv_cache_p"] + k_cache_d, v_cache_d = run["paged_kv_cache_d"] + v_cache_p_unequal = make_padded_paged_kv_view(v_cache_p, "NHD") + v_cache_d_unequal = make_padded_paged_kv_view(v_cache_d, "NHD") + wrapper = BatchPODWithPagedKVCacheWrapper( + torch.empty(128 * 1024 * 1024, dtype=torch.uint8, device="cuda"), "NHD" + ) + wrapper.plan( + plan["qo_indptr_p"], + plan["kv_indptr_p"], + plan["kv_indices_p"], + plan["last_page_len_p"], + plan["qo_indptr_d"], + plan["kv_indptr_d"], + plan["kv_indices_d"], + plan["last_page_len_d"], + plan["num_qo_heads"], + plan["num_kv_heads"], + plan["head_dim"], + plan["page_size"], + q_data_type=plan["q_data_type"], + kv_data_type=plan["kv_data_type"], + ) + plan_info_p = tuple(wrapper._plan_info_p) + plan_info_d = tuple(wrapper._plan_info_d) + planning_module = wrapper._cached_module + equal_output = wrapper.run( + run["q_p"], + (k_cache_p, v_cache_p), + run["q_d"], + (k_cache_d, v_cache_d), + causal_p=False, + ) + fused_module_misses = get_batch_pod_module.cache_info().misses + unequal_output = wrapper.run( + run["q_p"], + (k_cache_p, v_cache_p_unequal), + run["q_d"], + (k_cache_d, v_cache_d_unequal), + causal_p=False, + ) + assert get_batch_pod_module.cache_info().misses == fused_module_misses + assert wrapper._cached_module is planning_module + assert tuple(wrapper._plan_info_p) == plan_info_p + assert tuple(wrapper._plan_info_d) == plan_info_d + + expected = batch_pod_with_paged_kv_cache_run_trace.reference( + run["q_p"], + (k_cache_p, v_cache_p), + run["q_d"], + (k_cache_d, v_cache_d), + ) + _check( + batch_pod_with_paged_kv_cache_run_trace, + expected, + equal_output, + atol=1e-3, + rtol=1e-3, + ) + _check( + batch_pod_with_paged_kv_cache_run_trace, + expected, + unequal_output, + atol=1e-3, + rtol=1e-3, + ) diff --git a/tests/trace/test_pod_with_paged_kv_cache_run_reference_correctness.py b/tests/trace/test_pod_with_paged_kv_cache_run_reference_correctness.py index 1fa625fc1b2..04c427b2682 100644 --- a/tests/trace/test_pod_with_paged_kv_cache_run_reference_correctness.py +++ b/tests/trace/test_pod_with_paged_kv_cache_run_reference_correctness.py @@ -6,6 +6,7 @@ from tests.trace.reference_utils import ( _check, ) +from tests.test_helpers.paged_kv import make_padded_paged_kv_view @pytest.mark.parametrize( @@ -86,3 +87,82 @@ def test_pod_with_paged_kv_cache_run_reference_correctness(shape_kwargs): ) if torch.cuda.is_available(): torch.cuda.synchronize() + + +def test_pod_independent_paged_kv_strides(): + """Reuse one POD plan and fused module for equal and unequal decode KV.""" + from flashinfer.pod import PODWithPagedKVCacheWrapper, get_pod_module + from flashinfer.trace.templates.attention import pod_with_paged_kv_cache_run_trace + + inputs = pod_with_paged_kv_cache_run_trace.init( + device="cuda", + prefill_len=8, + decode_batch_size=1, + num_pages=2, + num_qo_heads=8, + num_kv_heads=2, + head_dim=64, + page_size=8, + ) + plan = inputs["plan"] + run = inputs["run"] + k_cache, v_cache = run["paged_kv_cache_d"] + v_cache_unequal = make_padded_paged_kv_view(v_cache, "NHD") + wrapper = PODWithPagedKVCacheWrapper( + torch.empty(64 * 1024 * 1024, dtype=torch.uint8, device="cuda"), "NHD" + ) + wrapper.plan( + plan["kv_indptr"], + plan["kv_indices"], + plan["kv_last_page_len"], + plan["num_qo_heads"], + plan["num_kv_heads"], + plan["head_dim"], + plan["page_size"], + q_data_type=plan["q_data_type"], + kv_data_type=plan["kv_data_type"], + ) + plan_info = tuple(wrapper._plan_info) + planning_module = wrapper._cached_module + equal_output = wrapper.run( + run["q_p"], + run["k_p"], + run["v_p"], + run["q_d"], + (k_cache, v_cache), + causal_p=True, + ) + fused_module_misses = get_pod_module.cache_info().misses + unequal_output = wrapper.run( + run["q_p"], + run["k_p"], + run["v_p"], + run["q_d"], + (k_cache, v_cache_unequal), + causal_p=True, + ) + assert get_pod_module.cache_info().misses == fused_module_misses + assert wrapper._cached_module is planning_module + assert tuple(wrapper._plan_info) == plan_info + + expected = pod_with_paged_kv_cache_run_trace.reference( + run["q_p"], + run["k_p"], + run["v_p"], + run["q_d"], + (k_cache, v_cache), + ) + _check( + pod_with_paged_kv_cache_run_trace, + expected, + equal_output, + atol=1e-3, + rtol=1e-3, + ) + _check( + pod_with_paged_kv_cache_run_trace, + expected, + unequal_output, + atol=1e-3, + rtol=1e-3, + ) diff --git a/tests/trace/test_var_block_sparse_run_reference_correctness.py b/tests/trace/test_var_block_sparse_run_reference_correctness.py index 480099002a1..7504ff0ddb7 100644 --- a/tests/trace/test_var_block_sparse_run_reference_correctness.py +++ b/tests/trace/test_var_block_sparse_run_reference_correctness.py @@ -6,6 +6,7 @@ from tests.trace.reference_utils import ( _check, ) +from tests.test_helpers.paged_kv import get_closure_value, make_padded_view @pytest.mark.parametrize( @@ -80,3 +81,84 @@ def test_var_block_sparse_run_reference_correctness(shape_kwargs): ) if torch.cuda.is_available(): torch.cuda.synchronize() + + +def test_var_block_sparse_lazy_stride_router(monkeypatch): + """Normalize unequal HND inputs onto the shared routed FA2 namespace.""" + from flashinfer import VariableBlockSparseAttentionWrapper + from flashinfer.trace.templates.attention import ( + variable_block_sparse_attention_run_trace, + ) + + inputs = variable_block_sparse_attention_run_trace.init( + device="cuda", + qo_len=32, + kv_len=32, + num_qo_heads=8, + num_kv_heads=2, + head_dim=64, + ) + R = C = 16 + M, num_qo_heads, head_dim = inputs["q"].shape + N, num_kv_heads, _ = inputs["k"].shape + block_mask_map = torch.ones( + num_kv_heads, M // R, N // C, dtype=torch.bool, device="cuda" + ) + block_row_sz = torch.full( + (num_kv_heads, M // R), R, dtype=torch.int32, device="cuda" + ) + block_col_sz = torch.full( + (num_kv_heads, N // C), C, dtype=torch.int32, device="cuda" + ) + q_hnd = inputs["q"].transpose(0, 1).contiguous() + k_hnd = inputs["k"].transpose(0, 1).contiguous() + v_hnd = inputs["v"].transpose(0, 1).contiguous() + v_hnd_unequal = make_padded_view(v_hnd, 1) + assert k_hnd.stride() == v_hnd.stride() + assert k_hnd.stride() != v_hnd_unequal.stride() + + wrapper = VariableBlockSparseAttentionWrapper( + torch.empty(128 * 1024 * 1024, device="cuda"), backend="fa2" + ) + wrapper.plan( + block_mask_map=block_mask_map, + block_row_sz=block_row_sz, + block_col_sz=block_col_sz, + num_qo_heads=num_qo_heads, + num_kv_heads=num_kv_heads, + head_dim=head_dim, + q_data_type=torch.float16, + kv_data_type=torch.float16, + ) + plan_info = tuple(wrapper._plan_info) + routed_module = wrapper._cached_module + holder = get_closure_value(routed_module.paged_run, "lazy_independent_module") + + def unexpected_supplement_load(): + raise AssertionError( + "variable-block sparse must normalize HND tensors before paged routing" + ) + + monkeypatch.setattr(holder, "get", unexpected_supplement_load) + equal_output = wrapper.run(q_hnd, k_hnd, v_hnd) + unequal_input_output = wrapper.run(q_hnd, k_hnd, v_hnd_unequal) + assert wrapper._cached_module is routed_module + assert tuple(wrapper._plan_info) == plan_info + + expected = variable_block_sparse_attention_run_trace.reference( + inputs["q"], inputs["k"], inputs["v"] + ) + _check( + variable_block_sparse_attention_run_trace, + expected, + equal_output.transpose(0, 1), + atol=1e-2, + rtol=1e-2, + ) + _check( + variable_block_sparse_attention_run_trace, + expected, + unequal_input_output.transpose(0, 1), + atol=1e-2, + rtol=1e-2, + ) From 3e2a84f0d2ff6077f4be5fd14de351f1ff7f58db Mon Sep 17 00:00:00 2001 From: mingyangw Date: Thu, 10 Sep 2026 21:34:48 +0000 Subject: [PATCH 3/6] test(attention): exercise split-KV in stride router plan reuse --- tests/attention/test_batch_prefill_kernels.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/attention/test_batch_prefill_kernels.py b/tests/attention/test_batch_prefill_kernels.py index 48dafc5212f..19f0d80421a 100644 --- a/tests/attention/test_batch_prefill_kernels.py +++ b/tests/attention/test_batch_prefill_kernels.py @@ -473,7 +473,7 @@ def test_batch_prefill_lazy_stride_router_plan_reuse( causal=True, q_data_type=torch.bfloat16, kv_data_type=torch.bfloat16, - fixed_split_size=None if disable_split_kv else 64, + fixed_split_size=None if disable_split_kv else 2, disable_split_kv=disable_split_kv, ) plan_info = tuple(wrapper._plan_info) From 20c7677b323c5730401a4a6e6f826742e29ce6c1 Mon Sep 17 00:00:00 2001 From: mingyangw Date: Fri, 11 Sep 2026 06:14:57 +0000 Subject: [PATCH 4/6] docs(attention): document paged KV stride prewarm parameters --- flashinfer/cascade.py | 22 ++++++++++++++++++++-- flashinfer/decode.py | 6 ++++++ flashinfer/prefill.py | 6 ++++++ 3 files changed, 32 insertions(+), 2 deletions(-) diff --git a/flashinfer/cascade.py b/flashinfer/cascade.py index a21dc7163e7..97e2f316279 100644 --- a/flashinfer/cascade.py +++ b/flashinfer/cascade.py @@ -518,7 +518,16 @@ def plan( @flashinfer_api def prewarm_paged_kv_stride_variant(self, variant: str = "independent") -> None: - r"""Prewarm every level's lazy paged-KV-stride variant after :meth:`plan`.""" + r"""Prewarm every level's lazy paged-KV-stride variant after :meth:`plan`. + + Call this method before CUDA graph capture. + + Parameters + ---------- + variant : str + The paged-KV-stride variant to prewarm. The only supported value is + ``"independent"`` (the default), for K and V with different data strides. + """ for wrapper in self._batch_prefill_wrappers: wrapper.prewarm_paged_kv_stride_variant(variant) @@ -1010,7 +1019,16 @@ def begin_forward( @flashinfer_api def prewarm_paged_kv_stride_variant(self, variant: str = "independent") -> None: - r"""Prewarm the underlying prefill wrapper's paged-KV-stride variant.""" + r"""Prewarm the underlying prefill wrapper's paged-KV-stride variant. + + Call this method after :meth:`begin_forward` and before CUDA graph capture. + + Parameters + ---------- + variant : str + The paged-KV-stride variant to prewarm. The only supported value is + ``"independent"`` (the default), for K and V with different data strides. + """ self._batch_prefill_wrapper.prewarm_paged_kv_stride_variant(variant) @flashinfer_api diff --git a/flashinfer/decode.py b/flashinfer/decode.py index 3bf6c6c6ff5..d9e0513843d 100644 --- a/flashinfer/decode.py +++ b/flashinfer/decode.py @@ -2052,6 +2052,12 @@ def prewarm_paged_kv_stride_variant(self, variant: str = "independent") -> None: Call this method before CUDA graph capture when a planned standard FA2 tensor-core decode wrapper will receive K and V tensors with different data strides. + + Parameters + ---------- + variant : str + The paged-KV-stride variant to prewarm. The only supported value is + ``"independent"`` (the default), for K and V with different data strides. """ if ( getattr(self, "_plan_info", None) is None diff --git a/flashinfer/prefill.py b/flashinfer/prefill.py index 0501d27480d..facc18ed091 100755 --- a/flashinfer/prefill.py +++ b/flashinfer/prefill.py @@ -2850,6 +2850,12 @@ def prewarm_paged_kv_stride_variant(self, variant: str = "independent") -> None: Call this method before CUDA graph capture when a planned standard FA2 wrapper will receive K and V tensors with different data strides. + + Parameters + ---------- + variant : str + The paged-KV-stride variant to prewarm. The only supported value is + ``"independent"`` (the default), for K and V with different data strides. """ if ( getattr(self, "_plan_info", None) is None From c22ce52c465b7ce1373e7349d156cc1d203acc43 Mon Sep 17 00:00:00 2001 From: mingyangw Date: Fri, 11 Sep 2026 22:20:05 +0000 Subject: [PATCH 5/6] fix(attention): forward custom prefill stride options Pass stride mode and module surface through the cached custom prefill adapter. Add coverage for legacy defaults and explicit specialization, and pin the FA2 shared-prefix router test to its intended backend. --- flashinfer/prefill.py | 6 ++ tests/attention/test_shared_prefix_kernels.py | 13 +++- tests/jit/test_attention_utils.py | 60 +++++++++++++++++++ 3 files changed, 78 insertions(+), 1 deletion(-) diff --git a/flashinfer/prefill.py b/flashinfer/prefill.py index facc18ed091..ccfb9e019e8 100755 --- a/flashinfer/prefill.py +++ b/flashinfer/prefill.py @@ -39,6 +39,8 @@ setup_cubin_loader, ) from .jit.attention.modules import ( + BatchPrefillModuleSurface, + BatchPrefillPagedKVStrideMode, _gen_batch_prefill_independent_paged_module, _gen_batch_prefill_primary_module, ) @@ -202,6 +204,8 @@ def get_customize_batch_prefill_module( use_logits_soft_cap: bool = False, use_fp16_qk_reduction: bool = False, fp8_enabled: bool = False, + paged_kv_stride_mode: BatchPrefillPagedKVStrideMode = "runtime", + module_surface: BatchPrefillModuleSurface = "full", ): return gen_customize_batch_prefill_module( backend, @@ -223,6 +227,8 @@ def get_customize_batch_prefill_module( use_logits_soft_cap, use_fp16_qk_reduction, fp8_enabled, + paged_kv_stride_mode=paged_kv_stride_mode, + module_surface=module_surface, ).build_and_load() diff --git a/tests/attention/test_shared_prefix_kernels.py b/tests/attention/test_shared_prefix_kernels.py index 717be73f260..f3a5ce3fa7e 100644 --- a/tests/attention/test_shared_prefix_kernels.py +++ b/tests/attention/test_shared_prefix_kernels.py @@ -231,8 +231,17 @@ def test_batch_attention_with_shared_prefix_paged_kv_cache( torch.testing.assert_close(o_multi_level, o_two_level, rtol=1e-3, atol=1e-3) -def test_shared_prefix_and_cascade_lazy_stride_router(): +def test_shared_prefix_and_cascade_lazy_stride_router(monkeypatch): """Share one routed FA2 namespace across cascade and shared-prefix prefill.""" + wrapper_cls = flashinfer.BatchPrefillWithPagedKVCacheWrapper + + def fa2_wrapper(*args, **kwargs): + return wrapper_cls(*args, **kwargs, backend="fa2") + + # Cascade defaults to auto, which selects FA3 on Hopper. This test covers FA2. + monkeypatch.setattr( + flashinfer.cascade, "BatchPrefillWithPagedKVCacheWrapper", fa2_wrapper + ) torch.manual_seed(42) batch_size, qo_len, shared_kv_len, unique_kv_len = 2, 8, 16, 16 page_size = 16 @@ -301,6 +310,8 @@ def test_shared_prefix_and_cascade_lazy_stride_router(): head_dim, page_size, ) + assert all(wrapper._backend == "fa2" for wrapper in cascade._batch_prefill_wrappers) + assert shared_prefix._batch_prefill_wrapper._backend == "fa2" modules = [wrapper._cached_module for wrapper in cascade._batch_prefill_wrappers] assert modules[0] is modules[1] assert shared_prefix._batch_prefill_wrapper._cached_module is modules[0] diff --git a/tests/jit/test_attention_utils.py b/tests/jit/test_attention_utils.py index dbcd4f25b66..33b91d020d5 100644 --- a/tests/jit/test_attention_utils.py +++ b/tests/jit/test_attention_utils.py @@ -4,12 +4,72 @@ import pytest import torch +from flashinfer import prefill from flashinfer.jit import core from flashinfer.jit import env as jit_env from flashinfer.jit.attention import modules as attention_modules from flashinfer.jit.attention.modules import gen_customize_batch_prefill_module +@pytest.mark.parametrize( + "backend,mode", + [("fa2", None), ("fa2", "independent"), ("fa3", None), ("fa3", "runtime")], +) +def test_custom_prefill_adapter_preserves_stride_specialization( + tmp_path, monkeypatch, backend, mode +): + repo_root = Path(__file__).resolve().parents[2] + monkeypatch.setattr(core, "check_cuda_arch", lambda: None) + monkeypatch.setattr( + attention_modules.current_compilation_context, "TARGET_CUDA_ARCHS", {(9, 0)} + ) + monkeypatch.setattr(jit_env, "FLASHINFER_GEN_SRC_DIR", tmp_path / "generated") + monkeypatch.setattr(jit_env, "FLASHINFER_CSRC_DIR", repo_root / "csrc") + # Exercise the cached adapter and real generator; only compilation is stubbed. + monkeypatch.setattr(core.JitSpec, "build_and_load", lambda self: self) + uri = f"test_custom_prefill_adapter_{backend}_{mode}_{tmp_path.name}" + kwargs = ( + {} if mode is None else {"paged_kv_stride_mode": mode, "module_surface": "full"} + ) + spec = prefill.get_customize_batch_prefill_module( + backend, + uri, + torch.bfloat16, + torch.bfloat16, + torch.bfloat16, + torch.int32, + 128, + 128, + [], + [], + [], + [], + "DefaultAttention", + "struct DefaultAttention {};", + **kwargs, + ) + assert spec.name == uri + assert len(spec.sources) == 10 + generated_dir = tmp_path / "generated" / uri + suffix = "_sm90" if backend == "fa3" else "" + binding = (generated_dir / f"batch_prefill{suffix}_jit_binding.cu").read_text() + assert "TVM_FFI_DLL_EXPORT_TYPED_FUNC(paged_run," in binding + assert "TVM_FFI_DLL_EXPORT_TYPED_FUNC(ragged_run," in binding + if backend == "fa2": + expected_mode = "INDEPENDENT" if mode == "independent" else "RUNTIME" + config = (generated_dir / "batch_prefill_config.inc").read_text() + assert ( + f"#define PAGED_KV_STRIDE_MODE PAGED_KV_STRIDE_MODE_{expected_mode}" + in config + ) + for mask in range(4): + source = ( + generated_dir / f"batch_prefill_paged_kernel_mask_{mask}.cu" + ).read_text() + assert ("/*SAME_KV_STRIDES=*/true" in source) == (mode is None) + assert "/*SAME_KV_STRIDES=*/false" in source + + def test_batch_prefill_nvfp4_swa_paged_params_declares_sf_strides( tmp_path, monkeypatch ): From ec383decd36a79a975a2cc02dd66ee33ed97c572 Mon Sep 17 00:00:00 2001 From: mingyangw Date: Sat, 12 Sep 2026 00:04:44 +0000 Subject: [PATCH 6/6] test(attention): trim stride specialization coverage --- tests/attention/test_batch_prefill_kernels.py | 316 ++---------------- tests/attention/test_block_sparse.py | 45 --- tests/attention/test_shared_prefix_kernels.py | 112 ------- tests/attention/test_tensor_cores_decode.py | 181 ---------- tests/jit/test_attention_utils.py | 299 ++++------------- tests/jit/test_batch_prefill_lazy_loader.py | 86 ++--- tests/test_helpers/paged_kv.py | 56 +--- ...est_batch_pod_run_reference_correctness.py | 88 ----- ...aged_kv_cache_run_reference_correctness.py | 80 ----- ..._block_sparse_run_reference_correctness.py | 82 ----- 10 files changed, 115 insertions(+), 1230 deletions(-) diff --git a/tests/attention/test_batch_prefill_kernels.py b/tests/attention/test_batch_prefill_kernels.py index 19f0d80421a..b1cff9a3705 100644 --- a/tests/attention/test_batch_prefill_kernels.py +++ b/tests/attention/test_batch_prefill_kernels.py @@ -20,10 +20,7 @@ import pytest import torch from tests.test_helpers.jit_utils import gen_prefill_attention_modules -from tests.test_helpers.paged_kv import ( - get_closure_value, - make_padded_paged_kv_view, -) +from tests.test_helpers.paged_kv import make_padded_paged_kv_view import flashinfer from tests.test_helpers.test_helpers import assert_close_chunked, ref_single_prefill @@ -342,75 +339,8 @@ def test_batch_prefill_with_paged_kv_cache( _assert_no_ref_mismatch(mismatch_counts) -@pytest.mark.cuda -def test_batch_prefill_equal_kv_strides_bf16(): - """Exercise the equal-stride FA2 paged path for the GPT-OSS head shape.""" - torch.manual_seed(42) - qo_len = kv_len = 65 - page_size = 16 - num_qo_heads, num_kv_heads, head_dim = 64, 8, 64 - pages_per_request = (kv_len + page_size - 1) // page_size - - q = torch.randn( - qo_len, - num_qo_heads, - head_dim, - device="cuda", - dtype=torch.bfloat16, - ) - k = torch.randn( - pages_per_request, - page_size, - num_kv_heads, - head_dim, - device="cuda", - dtype=torch.bfloat16, - ) - v = torch.randn_like(k) - assert k.stride() == v.stride() - - qo_indptr = torch.tensor([0, qo_len], device="cuda", dtype=torch.int32) - kv_indptr = torch.tensor([0, pages_per_request], device="cuda", dtype=torch.int32) - kv_indices = torch.arange(pages_per_request, device="cuda", dtype=torch.int32) - last_page_len = torch.tensor( - [(kv_len - 1) % page_size + 1], device="cuda", dtype=torch.int32 - ) - - workspace = torch.empty(128 * 1024 * 1024, device="cuda", dtype=torch.uint8) - wrapper = flashinfer.BatchPrefillWithPagedKVCacheWrapper( - workspace, "NHD", backend="fa2" - ) - wrapper.plan( - qo_indptr, - kv_indptr, - kv_indices, - last_page_len, - num_qo_heads, - num_kv_heads, - head_dim, - page_size, - causal=True, - q_data_type=torch.bfloat16, - kv_data_type=torch.bfloat16, - ) - actual = wrapper.run(q, (k, v)) - expected = flashinfer.prefill.single_prefill_with_kv_cache( - q, - k.view(-1, num_kv_heads, head_dim)[:kv_len], - v.view(-1, num_kv_heads, head_dim)[:kv_len], - causal=True, - backend="fa2", - ) - - torch.testing.assert_close(actual.float(), expected.float(), rtol=1e-2, atol=1e-2) - - -@pytest.mark.parametrize("kv_layout", ["NHD", "HND"]) -@pytest.mark.parametrize("disable_split_kv", [False, True]) -@pytest.mark.parametrize("head_dim", [64, 128]) -def test_batch_prefill_lazy_stride_router_plan_reuse( - kv_layout, disable_split_kv, head_dim -): +@pytest.mark.parametrize("kv_layout,head_dim", [("NHD", 64), ("HND", 128)]) +def test_batch_prefill_lazy_stride_router_plan_reuse(kv_layout, head_dim): """Reuse one equal-primary plan across equal/unequal/equal paged runs.""" torch.manual_seed(42) batch_size, qo_len, kv_len, page_size = 2, 17, 97, 16 @@ -473,8 +403,7 @@ def test_batch_prefill_lazy_stride_router_plan_reuse( causal=True, q_data_type=torch.bfloat16, kv_data_type=torch.bfloat16, - fixed_split_size=None if disable_split_kv else 2, - disable_split_kv=disable_split_kv, + fixed_split_size=2, ) plan_info = tuple(wrapper._plan_info) outputs = [ @@ -483,34 +412,27 @@ def test_batch_prefill_lazy_stride_router_plan_reuse( assert tuple(wrapper._plan_info) == plan_info expected_batches = [] - group_size = num_qo_heads // num_kv_heads for batch_idx in range(batch_size): - q_i = q[batch_idx * qo_len : (batch_idx + 1) * qo_len].float() + q_i = q[batch_idx * qo_len : (batch_idx + 1) * qo_len] page_slice = slice( batch_idx * pages_per_request, (batch_idx + 1) * pages_per_request, ) - k_i = to_dense(k[page_slice])[:kv_len].repeat_interleave(group_size, dim=1) - v_i = to_dense(v_equal[page_slice])[:kv_len].repeat_interleave( - group_size, dim=1 - ) - logits = torch.einsum("qhd,khd->hqk", q_i, k_i.float()) * head_dim**-0.5 - q_pos = torch.arange(qo_len, device="cuda").unsqueeze(1) - k_pos = torch.arange(kv_len, device="cuda").unsqueeze(0) - logits.masked_fill_( - ~(k_pos <= q_pos + (kv_len - qo_len)).unsqueeze(0), float("-inf") - ) - expected_batches.append( - torch.einsum("hqk,khd->qhd", torch.softmax(logits, dim=-1), v_i.float()) + expected_i, _ = ref_single_prefill( + q_i, + to_dense(k[page_slice])[:kv_len], + to_dense(v_equal[page_slice])[:kv_len], + causal=True, ) + expected_batches.append(expected_i) expected = torch.cat(expected_batches) for output in outputs: - torch.testing.assert_close(output.float(), expected, rtol=2e-2, atol=2e-2) + torch.testing.assert_close(output, expected, rtol=2e-2, atol=2e-2) torch.testing.assert_close(outputs[0], outputs[1], rtol=1e-2, atol=1e-2) torch.testing.assert_close(outputs[0], outputs[2], rtol=1e-2, atol=1e-2) -def test_batch_prefill_lazy_stride_router_nvfp4(monkeypatch): +def test_batch_prefill_lazy_stride_router_nvfp4(): """Route on data strides while preserving independent NVFP4 scale strides.""" torch.manual_seed(42) batch_size, qo_len, kv_len, page_size = 1, 17, 33, 16 @@ -564,23 +486,14 @@ def test_batch_prefill_lazy_stride_router_nvfp4(monkeypatch): kv_data_type=torch.uint8, ) plan_info = tuple(wrapper._plan_info) - holder = get_closure_value( - wrapper._cached_module.paged_run, "lazy_independent_module" + equal_output = wrapper.run( + q, + (k_packed, v_packed), + kv_cache_sf=(k_sf_unequal, v_sf_unequal), + k_scale=k_scale.item(), + v_scale=v_scale.item(), ) - def unexpected_supplement_route(): - raise AssertionError("scale stride asymmetry must not select the supplement") - - with monkeypatch.context() as patch: - patch.setattr(holder, "get", unexpected_supplement_route) - equal_output = wrapper.run( - q, - (k_packed, v_packed), - kv_cache_sf=(k_sf_unequal, v_sf_unequal), - k_scale=k_scale.item(), - v_scale=v_scale.item(), - ) - wrapper.prewarm_paged_kv_stride_variant("independent") unequal_output = wrapper.run( q, @@ -589,201 +502,10 @@ def unexpected_supplement_route(): k_scale=k_scale.item(), v_scale=v_scale.item(), ) - assert holder.is_loaded assert tuple(wrapper._plan_info) == plan_info torch.testing.assert_close(unequal_output, equal_output, rtol=1e-3, atol=1e-3) -def test_batch_prefill_lazy_stride_router_ragged_primary(monkeypatch): - """Keep ragged execution on the full equal-primary module.""" - torch.manual_seed(42) - batch_size, qo_len, kv_len = 2, 7, 11 - num_qo_heads, num_kv_heads, head_dim = 4, 2, 64 - q = torch.randn( - batch_size * qo_len, - num_qo_heads, - head_dim, - device="cuda", - dtype=torch.float16, - ) - k = torch.randn( - batch_size * kv_len, - num_kv_heads, - head_dim, - device="cuda", - dtype=torch.float16, - ) - v = torch.randn_like(k) - qo_indptr = torch.arange(batch_size + 1, device="cuda", dtype=torch.int32) * qo_len - kv_indptr = torch.arange(batch_size + 1, device="cuda", dtype=torch.int32) * kv_len - wrapper = flashinfer.BatchPrefillWithRaggedKVCacheWrapper( - torch.empty(128 * 1024 * 1024, device="cuda", dtype=torch.uint8), - "NHD", - backend="fa2", - ) - wrapper.plan( - qo_indptr, - kv_indptr, - num_qo_heads, - num_kv_heads, - head_dim, - causal=True, - q_data_type=torch.float16, - kv_data_type=torch.float16, - ) - holder = get_closure_value( - wrapper._cached_module.paged_run, "lazy_independent_module" - ) - - def unexpected_supplement_route(): - raise AssertionError("ragged execution must remain on the primary module") - - monkeypatch.setattr(holder, "get", unexpected_supplement_route) - output = wrapper.run(q, k, v) - for batch_idx in range(batch_size): - query_slice = slice(batch_idx * qo_len, (batch_idx + 1) * qo_len) - kv_slice = slice(batch_idx * kv_len, (batch_idx + 1) * kv_len) - expected = flashinfer.single_prefill_with_kv_cache( - q[query_slice], k[kv_slice], v[kv_slice], causal=True, backend="fa2" - ) - torch.testing.assert_close(output[query_slice], expected, rtol=1e-3, atol=1e-3) - - -def _make_lazy_stride_router_graph_case(): - batch_size, qo_len, kv_len, page_size = 2, 17, 97, 16 - num_qo_heads, num_kv_heads, head_dim = 8, 2, 128 - pages_per_request = (kv_len + page_size - 1) // page_size - total_pages = batch_size * pages_per_request - q = torch.randn( - batch_size * qo_len, - num_qo_heads, - head_dim, - device="cuda", - dtype=torch.bfloat16, - ) - k = torch.randn( - total_pages, - page_size, - num_kv_heads, - head_dim, - device="cuda", - dtype=torch.bfloat16, - ) - v_equal = torch.randn_like(k) - v_unequal = make_padded_paged_kv_view(v_equal, "NHD") - qo_indptr = torch.arange(batch_size + 1, device="cuda", dtype=torch.int32) * qo_len - kv_indptr = ( - torch.arange(batch_size + 1, device="cuda", dtype=torch.int32) - * pages_per_request - ) - kv_indices = torch.arange(total_pages, device="cuda", dtype=torch.int32) - last_page_len = torch.full( - (batch_size,), - (kv_len - 1) % page_size + 1, - device="cuda", - dtype=torch.int32, - ) - wrapper = flashinfer.BatchPrefillWithPagedKVCacheWrapper( - torch.empty(128 * 1024 * 1024, device="cuda", dtype=torch.uint8), - "NHD", - backend="fa2", - ) - wrapper.plan( - qo_indptr, - kv_indptr, - kv_indices, - last_page_len, - num_qo_heads, - num_kv_heads, - head_dim, - page_size, - causal=True, - q_data_type=torch.bfloat16, - kv_data_type=torch.bfloat16, - fixed_split_size=64, - ) - return wrapper, q, k, v_equal, v_unequal - - -def test_batch_prefill_lazy_stride_router_cuda_graph(): - """A post-plan prewarm makes the unequal route capture/replay safe.""" - torch.manual_seed(42) - wrapper, q, k, v_equal, v_unequal = _make_lazy_stride_router_graph_case() - plan_info = tuple(wrapper._plan_info) - equal_output = wrapper.run(q, (k, v_equal), enable_pdl=False) - wrapper.prewarm_paged_kv_stride_variant("independent") - - graph_output = torch.empty_like(equal_output) - warmup_stream = torch.cuda.Stream() - warmup_stream.wait_stream(torch.cuda.current_stream()) - with torch.cuda.stream(warmup_stream): - for _ in range(3): - wrapper.run( - q, - (k, v_unequal), - out=graph_output, - enable_pdl=False, - ) - torch.cuda.current_stream().wait_stream(warmup_stream) - graph = torch.cuda.CUDAGraph() - with torch.cuda.graph(graph): - wrapper.run( - q, - (k, v_unequal), - out=graph_output, - enable_pdl=False, - ) - graph_output.fill_(float("nan")) - graph.replay() - torch.cuda.synchronize() - assert torch.isfinite(graph_output).all() - assert tuple(wrapper._plan_info) == plan_info - torch.testing.assert_close(graph_output, equal_output, rtol=1e-2, atol=1e-2) - - -def test_batch_prefill_lazy_stride_router_torch_compile(): - """Dynamo guards both stride families without rebuilding on equal return.""" - torch.manual_seed(42) - wrapper, q, k, v_equal, v_unequal = _make_lazy_stride_router_graph_case() - wrapper.prewarm_paged_kv_stride_variant("independent") - expected = wrapper.run(q, (k, v_equal), enable_pdl=False) - torch._dynamo.reset() - from torch._dynamo.utils import guard_failures - - guard_failures.clear() - compiled_graphs = [] - - def counting_backend(graph_module, _example_inputs): - compiled_graphs.append(graph_module) - return graph_module.forward - - def run_for_compile(q_arg, k_arg, v_arg, out_arg): - return wrapper.run( - q_arg, - (k_arg, v_arg), - out=out_arg, - enable_pdl=False, - ).clone() - - compiled_run = torch.compile( - run_for_compile, - backend=counting_backend, - fullgraph=False, - dynamic=False, - ) - graph_counts = [] - for v_cache in (v_equal, v_equal, v_unequal, v_unequal, v_equal): - actual = compiled_run(q, k, v_cache, torch.empty_like(expected)) - torch.testing.assert_close(actual, expected, rtol=1e-2, atol=1e-2) - graph_counts.append(len(compiled_graphs)) - failures = [str(reason) for reason in guard_failures[run_for_compile.__code__]] - assert len(failures) == 1 and "stride mismatch" in failures[0] - assert len(compiled_graphs) == 1 - assert graph_counts[1] == graph_counts[0] - assert graph_counts[3] == graph_counts[2] - assert graph_counts[4] == graph_counts[3] - - @pytest.mark.parametrize("causal", [False, True]) @pytest.mark.parametrize("pos_encoding_mode", ["NONE", "ROPE_LLAMA"]) def test_batch_prefill_with_paged_kv_cache_head_dim_512( diff --git a/tests/attention/test_block_sparse.py b/tests/attention/test_block_sparse.py index 7b66ba26b2f..83e0d02b867 100644 --- a/tests/attention/test_block_sparse.py +++ b/tests/attention/test_block_sparse.py @@ -30,7 +30,6 @@ gen_decode_attention_modules, gen_prefill_attention_modules, ) -from tests.test_helpers.paged_kv import make_padded_view import flashinfer from flashinfer.cutile.cutile_common import is_cuda_tile_available @@ -238,50 +237,6 @@ def test_block_sparse_attention( ) -def test_block_sparse_lazy_stride_router(): - """Run one tensor-core block-sparse plan with equal and unequal KV strides.""" - set_seed(42) - M = N = 32 - R = C = 16 - num_qo_heads, num_kv_heads, head_dim = 4, 2, 128 - indptr = torch.tensor([0, 2, 4], dtype=torch.int32, device="cuda") - indices = torch.tensor([0, 1, 0, 1], dtype=torch.int32, device="cuda") - q = torch.randn(M, num_qo_heads, head_dim, dtype=torch.float16, device="cuda") - k = torch.randn(N, num_kv_heads, head_dim, dtype=torch.float16, device="cuda") - v_equal = torch.randn_like(k) - v_unequal = make_padded_view(v_equal, 1) - assert k.stride() == v_equal.stride() - assert k.stride() != v_unequal.stride() - - wrapper = flashinfer.sparse.BlockSparseAttentionWrapper( - torch.empty(128 * 1024 * 1024, dtype=torch.uint8, device="cuda"), - backend="fa2", - ) - wrapper.plan( - indptr, - indices, - M, - N, - R, - C, - num_qo_heads, - num_kv_heads, - head_dim, - q_data_type=torch.float16, - kv_data_type=torch.float16, - ) - plan_info = tuple(wrapper._plan_info) - routed_module = wrapper._cached_module - equal_output = wrapper.run(q, k, v_equal) - unequal_output = wrapper.run(q, k, v_unequal) - assert wrapper._cached_module is routed_module - assert tuple(wrapper._plan_info) == plan_info - - expected = flashinfer.single_prefill_with_kv_cache(q, k, v_equal, backend="fa2") - torch.testing.assert_close(equal_output, expected, atol=1e-2, rtol=1e-3) - torch.testing.assert_close(unequal_output, expected, atol=1e-2, rtol=1e-3) - - def _ref_attention( q: torch.Tensor, # [gqa_group_size, qo_len, head_dim] k: torch.Tensor, # [1, kv_len, head_dim] diff --git a/tests/attention/test_shared_prefix_kernels.py b/tests/attention/test_shared_prefix_kernels.py index f3a5ce3fa7e..30aee0dc388 100644 --- a/tests/attention/test_shared_prefix_kernels.py +++ b/tests/attention/test_shared_prefix_kernels.py @@ -20,7 +20,6 @@ gen_decode_attention_modules, gen_prefill_attention_modules, ) -from tests.test_helpers.paged_kv import make_padded_paged_kv_view import flashinfer from flashinfer.utils import has_flashinfer_jit_cache @@ -231,117 +230,6 @@ def test_batch_attention_with_shared_prefix_paged_kv_cache( torch.testing.assert_close(o_multi_level, o_two_level, rtol=1e-3, atol=1e-3) -def test_shared_prefix_and_cascade_lazy_stride_router(monkeypatch): - """Share one routed FA2 namespace across cascade and shared-prefix prefill.""" - wrapper_cls = flashinfer.BatchPrefillWithPagedKVCacheWrapper - - def fa2_wrapper(*args, **kwargs): - return wrapper_cls(*args, **kwargs, backend="fa2") - - # Cascade defaults to auto, which selects FA3 on Hopper. This test covers FA2. - monkeypatch.setattr( - flashinfer.cascade, "BatchPrefillWithPagedKVCacheWrapper", fa2_wrapper - ) - torch.manual_seed(42) - batch_size, qo_len, shared_kv_len, unique_kv_len = 2, 8, 16, 16 - page_size = 16 - num_qo_heads, num_kv_heads, head_dim = 4, 2, 128 - q = torch.randn( - batch_size * qo_len, - num_qo_heads, - head_dim, - device="cuda", - dtype=torch.float16, - ) - k_cache = torch.randn( - 1 + batch_size, - page_size, - num_kv_heads, - head_dim, - device="cuda", - dtype=torch.float16, - ) - v_cache_equal = torch.randn_like(k_cache) - v_cache_unequal = make_padded_paged_kv_view(v_cache_equal, "NHD") - k_shared = k_cache[0, :shared_kv_len] - v_shared = v_cache_equal[0, :shared_kv_len] - - qo_indptr_top = torch.tensor( - [0, batch_size * qo_len], dtype=torch.int32, device="cuda" - ) - qo_indptr_bottom = ( - torch.arange(batch_size + 1, dtype=torch.int32, device="cuda") * qo_len - ) - shared_kv_indptr = torch.tensor([0, 1], dtype=torch.int32, device="cuda") - shared_kv_indices = torch.tensor([0], dtype=torch.int32, device="cuda") - shared_last_page_len = torch.tensor( - [shared_kv_len], dtype=torch.int32, device="cuda" - ) - unique_kv_indptr = torch.arange(batch_size + 1, dtype=torch.int32, device="cuda") - unique_kv_indices = torch.arange( - 1, 1 + batch_size, dtype=torch.int32, device="cuda" - ) - unique_last_page_len = torch.full( - (batch_size,), unique_kv_len, dtype=torch.int32, device="cuda" - ) - workspace = torch.empty(128 * 1024 * 1024, dtype=torch.uint8, device="cuda") - cascade = flashinfer.MultiLevelCascadeAttentionWrapper(2, workspace, "NHD") - cascade.plan( - [qo_indptr_top, qo_indptr_bottom], - [shared_kv_indptr, unique_kv_indptr], - [shared_kv_indices, unique_kv_indices], - [shared_last_page_len, unique_last_page_len], - num_qo_heads, - num_kv_heads, - head_dim, - page_size, - causal=False, - ) - shared_prefix = flashinfer.BatchPrefillWithSharedPrefixPagedKVCacheWrapper( - workspace, "NHD" - ) - shared_prefix.begin_forward( - qo_indptr_bottom, - unique_kv_indptr, - unique_kv_indices, - unique_last_page_len, - num_qo_heads, - num_kv_heads, - head_dim, - page_size, - ) - assert all(wrapper._backend == "fa2" for wrapper in cascade._batch_prefill_wrappers) - assert shared_prefix._batch_prefill_wrapper._backend == "fa2" - modules = [wrapper._cached_module for wrapper in cascade._batch_prefill_wrappers] - assert modules[0] is modules[1] - assert shared_prefix._batch_prefill_wrapper._cached_module is modules[0] - cascade_plans = [ - tuple(wrapper._plan_info) for wrapper in cascade._batch_prefill_wrappers - ] - shared_prefix_plan = tuple(shared_prefix._batch_prefill_wrapper._plan_info) - - equal_cache = (k_cache, v_cache_equal) - unequal_cache = (k_cache, v_cache_unequal) - cascade_equal = cascade.run(q, equal_cache) - prefix_equal = shared_prefix.forward( - q, k_shared, v_shared, equal_cache, causal=False - ) - cascade.prewarm_paged_kv_stride_variant("independent") - shared_prefix.prewarm_paged_kv_stride_variant("independent") - cascade_unequal = cascade.run(q, unequal_cache) - prefix_unequal = shared_prefix.forward( - q, k_shared, v_shared, unequal_cache, causal=False - ) - - assert [ - tuple(wrapper._plan_info) for wrapper in cascade._batch_prefill_wrappers - ] == cascade_plans - assert tuple(shared_prefix._batch_prefill_wrapper._plan_info) == shared_prefix_plan - torch.testing.assert_close(cascade_equal, prefix_equal, rtol=1e-3, atol=1e-3) - torch.testing.assert_close(cascade_unequal, prefix_equal, rtol=1e-3, atol=1e-3) - torch.testing.assert_close(prefix_unequal, prefix_equal, rtol=1e-3, atol=1e-3) - - @pytest.mark.parametrize("seed", [0]) @pytest.mark.parametrize("num_tries", [50]) def test_merge_state_in_place_with_mask(seed, num_tries): diff --git a/tests/attention/test_tensor_cores_decode.py b/tests/attention/test_tensor_cores_decode.py index 2047fe6e3c0..19db15a640f 100644 --- a/tests/attention/test_tensor_cores_decode.py +++ b/tests/attention/test_tensor_cores_decode.py @@ -20,7 +20,6 @@ gen_decode_attention_modules, gen_prefill_attention_modules, ) -from tests.test_helpers.paged_kv import make_padded_paged_kv_view from functools import partial import flashinfer from flashinfer.utils import has_flashinfer_jit_cache @@ -194,186 +193,6 @@ def test_batch_decode_tensor_cores( torch.testing.assert_close(lse, lse_tensor_cores, rtol=1e-3, atol=1e-3) -@pytest.mark.cuda -def test_batch_decode_tensor_cores_equal_kv_strides_bf16_group_size_5(): - """Exercise the equal-stride FA2 paged path for the Llama-4 head ratio.""" - torch.manual_seed(42) - batch_size, kv_len, page_size = 2, 97, 16 - num_qo_heads, num_kv_heads, head_dim = 40, 8, 128 - pages_per_request = (kv_len + page_size - 1) // page_size - total_pages = batch_size * pages_per_request - - q = torch.randn( - batch_size, - num_qo_heads, - head_dim, - device="cuda", - dtype=torch.bfloat16, - ) - k = torch.randn( - total_pages, - page_size, - num_kv_heads, - head_dim, - device="cuda", - dtype=torch.bfloat16, - ) - v = torch.randn_like(k) - assert k.stride() == v.stride() - - kv_indptr = ( - torch.arange(batch_size + 1, device="cuda", dtype=torch.int32) - * pages_per_request - ) - kv_indices = torch.arange(total_pages, device="cuda", dtype=torch.int32) - last_page_len = torch.full( - (batch_size,), - (kv_len - 1) % page_size + 1, - device="cuda", - dtype=torch.int32, - ) - - workspace = torch.empty(128 * 1024 * 1024, device="cuda", dtype=torch.uint8) - wrapper = flashinfer.BatchDecodeWithPagedKVCacheWrapper( - workspace, "NHD", use_tensor_cores=True, backend="fa2" - ) - wrapper.plan( - kv_indptr, - kv_indices, - last_page_len, - num_qo_heads, - num_kv_heads, - head_dim, - page_size, - q_data_type=torch.bfloat16, - kv_data_type=torch.bfloat16, - ) - actual = wrapper.run(q, (k, v)) - - k_dense = k.view(batch_size, pages_per_request * page_size, num_kv_heads, head_dim)[ - :, :kv_len - ] - v_dense = v.view(batch_size, pages_per_request * page_size, num_kv_heads, head_dim)[ - :, :kv_len - ] - group_size = num_qo_heads // num_kv_heads - k_dense = k_dense.repeat_interleave(group_size, dim=2).float() - v_dense = v_dense.repeat_interleave(group_size, dim=2).float() - logits = torch.einsum("bhd,bkhd->bhk", q.float(), k_dense) * head_dim**-0.5 - expected = torch.einsum("bhk,bkhd->bhd", torch.softmax(logits, dim=-1), v_dense) - - torch.testing.assert_close(actual.float(), expected, rtol=1e-2, atol=1e-2) - - -def test_batch_decode_lazy_stride_router(): - """Prewarm and capture the independently strided tensor-core decode path.""" - torch.manual_seed(42) - batch_size, kv_len, page_size = 2, 97, 16 - num_qo_heads, num_kv_heads, head_dim = 40, 8, 128 - pages_per_request = (kv_len + page_size - 1) // page_size - total_pages = batch_size * pages_per_request - - q = torch.randn( - batch_size, - num_qo_heads, - head_dim, - device="cuda", - dtype=torch.bfloat16, - ) - k = torch.randn( - total_pages, - page_size, - num_kv_heads, - head_dim, - device="cuda", - dtype=torch.bfloat16, - ) - v_equal = torch.randn_like(k) - v_unequal = make_padded_paged_kv_view(v_equal, "NHD") - assert k.stride() == v_equal.stride() - assert k.stride() != v_unequal.stride() - - kv_indptr = ( - torch.arange(batch_size + 1, device="cuda", dtype=torch.int32) - * pages_per_request - ) - kv_indices = torch.arange(total_pages, device="cuda", dtype=torch.int32) - last_page_len = torch.full( - (batch_size,), - (kv_len - 1) % page_size + 1, - device="cuda", - dtype=torch.int32, - ) - workspace = torch.empty(128 * 1024 * 1024, device="cuda", dtype=torch.uint8) - wrapper = flashinfer.BatchDecodeWithPagedKVCacheWrapper( - workspace, - "NHD", - use_cuda_graph=True, - use_tensor_cores=True, - paged_kv_indptr_buffer=torch.empty_like(kv_indptr), - paged_kv_indices_buffer=torch.empty_like(kv_indices), - paged_kv_last_page_len_buffer=torch.empty_like(last_page_len), - backend="fa2", - ) - wrapper.plan( - kv_indptr, - kv_indices, - last_page_len, - num_qo_heads, - num_kv_heads, - head_dim, - page_size, - q_data_type=torch.bfloat16, - kv_data_type=torch.bfloat16, - ) - plan_info = tuple(wrapper._plan_info) - equal_output = wrapper.run(q, (k, v_equal), enable_pdl=False).clone() - wrapper.prewarm_paged_kv_stride_variant("independent") - unequal_output = wrapper.run(q, (k, v_unequal), enable_pdl=False).clone() - - graph_output = torch.empty_like(equal_output) - warmup_stream = torch.cuda.Stream() - warmup_stream.wait_stream(torch.cuda.current_stream()) - with torch.cuda.stream(warmup_stream): - for _ in range(3): - wrapper.run( - q, - (k, v_unequal), - out=graph_output, - enable_pdl=False, - ) - torch.cuda.current_stream().wait_stream(warmup_stream) - - graph = torch.cuda.CUDAGraph() - with torch.cuda.graph(graph): - wrapper.run( - q, - (k, v_unequal), - out=graph_output, - enable_pdl=False, - ) - graph_output.fill_(float("nan")) - graph.replay() - torch.cuda.synchronize() - assert tuple(wrapper._plan_info) == plan_info - - k_dense = k.view(batch_size, pages_per_request * page_size, num_kv_heads, head_dim)[ - :, :kv_len - ] - v_dense = v_equal.view( - batch_size, pages_per_request * page_size, num_kv_heads, head_dim - )[:, :kv_len] - group_size = num_qo_heads // num_kv_heads - k_dense = k_dense.repeat_interleave(group_size, dim=2).float() - v_dense = v_dense.repeat_interleave(group_size, dim=2).float() - logits = torch.einsum("bhd,bkhd->bhk", q.float(), k_dense) * head_dim**-0.5 - expected = torch.einsum("bhk,bkhd->bhd", torch.softmax(logits, dim=-1), v_dense) - - torch.testing.assert_close(equal_output.float(), expected, rtol=1e-2, atol=1e-2) - torch.testing.assert_close(unequal_output.float(), expected, rtol=1e-2, atol=1e-2) - torch.testing.assert_close(graph_output.float(), expected, rtol=1e-2, atol=1e-2) - - @pytest.mark.parametrize("batch_size", [12, 17]) @pytest.mark.parametrize("kv_len", [54, 97, 512]) @pytest.mark.parametrize("page_size", [1, 8, 16]) diff --git a/tests/jit/test_attention_utils.py b/tests/jit/test_attention_utils.py index 33b91d020d5..cc5825c0d58 100644 --- a/tests/jit/test_attention_utils.py +++ b/tests/jit/test_attention_utils.py @@ -1,4 +1,3 @@ -import inspect from pathlib import Path import pytest @@ -11,23 +10,29 @@ from flashinfer.jit.attention.modules import gen_customize_batch_prefill_module +@pytest.fixture +def generated_sources(tmp_path, monkeypatch): + monkeypatch.setattr(core, "check_cuda_arch", lambda: None) + monkeypatch.setattr( + attention_modules.current_compilation_context, "TARGET_CUDA_ARCHS", {(9, 0)} + ) + monkeypatch.setattr(jit_env, "FLASHINFER_GEN_SRC_DIR", tmp_path) + monkeypatch.setattr( + jit_env, "FLASHINFER_CSRC_DIR", Path(__file__).resolve().parents[2] / "csrc" + ) + return tmp_path + + @pytest.mark.parametrize( "backend,mode", [("fa2", None), ("fa2", "independent"), ("fa3", None), ("fa3", "runtime")], ) def test_custom_prefill_adapter_preserves_stride_specialization( - tmp_path, monkeypatch, backend, mode + generated_sources, monkeypatch, backend, mode ): - repo_root = Path(__file__).resolve().parents[2] - monkeypatch.setattr(core, "check_cuda_arch", lambda: None) - monkeypatch.setattr( - attention_modules.current_compilation_context, "TARGET_CUDA_ARCHS", {(9, 0)} - ) - monkeypatch.setattr(jit_env, "FLASHINFER_GEN_SRC_DIR", tmp_path / "generated") - monkeypatch.setattr(jit_env, "FLASHINFER_CSRC_DIR", repo_root / "csrc") # Exercise the cached adapter and real generator; only compilation is stubbed. monkeypatch.setattr(core.JitSpec, "build_and_load", lambda self: self) - uri = f"test_custom_prefill_adapter_{backend}_{mode}_{tmp_path.name}" + uri = f"test_custom_prefill_adapter_{backend}_{mode}_{generated_sources.name}" kwargs = ( {} if mode is None else {"paged_kv_stride_mode": mode, "module_surface": "full"} ) @@ -50,7 +55,7 @@ def test_custom_prefill_adapter_preserves_stride_specialization( ) assert spec.name == uri assert len(spec.sources) == 10 - generated_dir = tmp_path / "generated" / uri + generated_dir = generated_sources / uri suffix = "_sm90" if backend == "fa3" else "" binding = (generated_dir / f"batch_prefill{suffix}_jit_binding.cu").read_text() assert "TVM_FFI_DLL_EXPORT_TYPED_FUNC(paged_run," in binding @@ -62,12 +67,6 @@ def test_custom_prefill_adapter_preserves_stride_specialization( f"#define PAGED_KV_STRIDE_MODE PAGED_KV_STRIDE_MODE_{expected_mode}" in config ) - for mask in range(4): - source = ( - generated_dir / f"batch_prefill_paged_kernel_mask_{mask}.cu" - ).read_text() - assert ("/*SAME_KV_STRIDES=*/true" in source) == (mode is None) - assert "/*SAME_KV_STRIDES=*/false" in source def test_batch_prefill_nvfp4_swa_paged_params_declares_sf_strides( @@ -125,41 +124,22 @@ def test_batch_prefill_nvfp4_swa_paged_params_declares_sf_strides( assert f"uint32_t {field}_stride_n;" in generated -def test_batch_prefill_generates_equal_and_independent_stride_specializations( - tmp_path, monkeypatch +@pytest.mark.parametrize( + "factory,suffix,source_count,variants", + [ + ("gen_batch_prefill_module", "", 10, {"true", "false"}), + ("_gen_batch_prefill_primary_module", "_kv_stride_equal", 10, {"true"}), + ( + "_gen_batch_prefill_independent_paged_module", + "_paged_kv_stride_independent", + 6, + {"false"}, + ), + ], +) +def test_batch_prefill_stride_module_surface( + generated_sources, factory, suffix, source_count, variants ): - repo_root = Path(__file__).resolve().parents[2] - monkeypatch.setattr(core, "check_cuda_arch", lambda: None) - monkeypatch.setattr( - attention_modules.current_compilation_context, - "TARGET_CUDA_ARCHS", - {(10, 3)}, - ) - monkeypatch.setattr(jit_env, "FLASHINFER_GEN_SRC_DIR", tmp_path / "generated") - monkeypatch.setattr(jit_env, "FLASHINFER_CSRC_DIR", repo_root / "csrc") - - public_parameters = inspect.signature( - attention_modules.gen_batch_prefill_module - ).parameters - assert list(public_parameters) == [ - "backend", - "dtype_q", - "dtype_kv", - "dtype_o", - "dtype_idx", - "head_dim_qk", - "head_dim_vo", - "pos_encoding_mode", - "use_sliding_window", - "use_logits_soft_cap", - "use_fp16_qk_reduction", - ] - assert all( - parameter.default is inspect.Parameter.empty - and parameter.kind is inspect.Parameter.POSITIONAL_OR_KEYWORD - for parameter in public_parameters.values() - ) - args = ( "fa2", torch.bfloat16, @@ -173,189 +153,31 @@ def test_batch_prefill_generates_equal_and_independent_stride_specializations( False, False, ) - base_uri = attention_modules.get_batch_prefill_uri(*args) - specs = { - "runtime_full": attention_modules.gen_batch_prefill_module(*args), - "equal_full": attention_modules._gen_batch_prefill_primary_module(*args), - "independent_full": ( - attention_modules._gen_batch_prefill_independent_full_module(*args) - ), - "independent_paged": ( - attention_modules._gen_batch_prefill_independent_paged_module(*args) - ), - } - - assert {variant: spec.name for variant, spec in specs.items()} == { - "runtime_full": base_uri, - "equal_full": f"{base_uri}_kv_stride_equal", - "independent_full": f"{base_uri}_kv_stride_independent", - "independent_paged": f"{base_uri}_paged_kv_stride_independent", - } - assert len({spec.name for spec in specs.values()}) == 4 - - full_sources = sorted( - [f"batch_prefill_paged_kernel_mask_{mask_mode}.cu" for mask_mode in range(4)] - + [f"batch_prefill_ragged_kernel_mask_{mask_mode}.cu" for mask_mode in range(4)] - + ["batch_prefill.cu", "batch_prefill_jit_binding.cu"] - ) - paged_sources = sorted( - [f"batch_prefill_paged_kernel_mask_{mask_mode}.cu" for mask_mode in range(4)] - + ["batch_prefill_paged.cu", "batch_prefill_paged_jit_binding.cu"] - ) - expected_sources = { - "runtime_full": full_sources, - "equal_full": full_sources, - "independent_full": full_sources, - "independent_paged": paged_sources, - } - expected_instantiations = { - "runtime_full": {"true": 3, "false": 3}, - "equal_full": {"true": 3, "false": 0}, - "independent_full": {"true": 0, "false": 3}, - "independent_paged": {"true": 0, "false": 3}, - } - expected_mode_macros = { - "runtime_full": "PAGED_KV_STRIDE_MODE_RUNTIME", - "equal_full": "PAGED_KV_STRIDE_MODE_EQUAL", - "independent_full": "PAGED_KV_STRIDE_MODE_INDEPENDENT", - "independent_paged": "PAGED_KV_STRIDE_MODE_INDEPENDENT", - } - - for variant, spec in specs.items(): - generated_dir = tmp_path / "generated" / spec.name - assert {source.parent for source in spec.sources} == {generated_dir} - assert ( - sorted(source.name for source in spec.sources) == expected_sources[variant] - ) - assert len(spec.sources) == (6 if variant == "independent_paged" else 10) - - config = (generated_dir / "batch_prefill_config.inc").read_text() - assert "#define PAGED_KV_STRIDE_MODE " + expected_mode_macros[variant] in config - paged_header = generated_dir / "batch_prefill_paged.cuh" - assert paged_header.exists() - assert paged_header not in spec.sources - paged_host_impl = paged_header.read_text() - assert "paged_k_cache.ndim()" in paged_host_impl - assert "route unequal layouts" in paged_host_impl - assert "through the independent paged module" in paged_host_impl - - is_paged_only = variant == "independent_paged" - host_name = "batch_prefill_paged.cu" if is_paged_only else "batch_prefill.cu" - binding_name = ( - "batch_prefill_paged_jit_binding.cu" - if is_paged_only - else "batch_prefill_jit_binding.cu" - ) - host = (generated_dir / host_name).read_text() - assert '#include "batch_prefill_paged.cuh"' in host - binding = (generated_dir / binding_name).read_text() - assert "TVM_FFI_DLL_EXPORT_TYPED_FUNC(paged_run" in binding - assert binding.count("TVM_FFI_DLL_EXPORT_TYPED_FUNC(") == ( - 1 if is_paged_only else 4 - ) - if is_paged_only: - for excluded_export in ("plan", "workspace_size", "ragged_run"): - assert f"TVM_FFI_DLL_EXPORT_TYPED_FUNC({excluded_export}" not in binding - - for mask_mode in range(4): - paged_text = ( - generated_dir / f"batch_prefill_paged_kernel_mask_{mask_mode}.cu" - ).read_text() - assert paged_text.count( - "template cudaError_t BatchPrefillWithPagedKVCacheDispatched<" - ) == sum(expected_instantiations[variant].values()) - for value, count in expected_instantiations[variant].items(): - assert paged_text.count(f"/*SAME_KV_STRIDES=*/{value}") == count - - ragged_path = ( - generated_dir / f"batch_prefill_ragged_kernel_mask_{mask_mode}.cu" - ) - if is_paged_only: - assert not ragged_path.exists() - else: - assert ( - ragged_path.read_text().count( - "template cudaError_t BatchPrefillWithRaggedKVCacheDispatched<" - ) - == 3 - ) - - for invalid_mode in ("runtime", "equal"): - invalid_uri = f"invalid_{invalid_mode}_paged_surface" - with pytest.raises( - ValueError, match="Unsupported batch-prefill mode/surface combination" - ): - gen_customize_batch_prefill_module( - "fa2", - invalid_uri, - torch.bfloat16, - torch.bfloat16, - torch.bfloat16, - torch.int32, - 64, - 64, - [], - [], - [], - [], - "DefaultAttention", - "struct DefaultAttention {};", - paged_kv_stride_mode=invalid_mode, - module_surface="paged", - ) - assert not (tmp_path / "generated" / invalid_uri).exists() - - -def test_gen_fa2_aot_uses_equal_primary_only(monkeypatch): - import flashinfer.aot as flashinfer_aot - - sentinel = object() - calls = [] - - def fake_primary(**kwargs): - calls.append(kwargs) - return sentinel - - def unexpected_public_full(**kwargs): - raise AssertionError("gen_fa2 must not put the runtime/full module in AOT") - - monkeypatch.setattr( - flashinfer_aot, "_gen_batch_prefill_primary_module", fake_primary - ) - monkeypatch.setattr( - flashinfer_aot, "gen_batch_prefill_module", unexpected_public_full + spec = getattr(attention_modules, factory)(*args) + assert spec.name == attention_modules.get_batch_prefill_uri(*args) + suffix + assert len(spec.sources) == source_count + for mask in range(4): + source = ( + generated_sources / spec.name / f"batch_prefill_paged_kernel_mask_{mask}.cu" + ).read_text() + for variant in ("true", "false"): + assert (f"/*SAME_KV_STRIDES=*/{variant}" in source) == (variant in variants) + binding_name = ( + "batch_prefill_paged_jit_binding.cu" + if source_count == 6 + else "batch_prefill_jit_binding.cu" ) - - specs = list( - flashinfer_aot.gen_fa2( - dtype_qo=torch.bfloat16, - dtype_kv=torch.bfloat16, - head_dim_qk=128, - head_dim_vo=128, - use_sliding_window=False, - use_logits_soft_cap=False, - prefill_only=True, + binding = (generated_sources / spec.name / binding_name).read_text() + assert "TVM_FFI_DLL_EXPORT_TYPED_FUNC(paged_run," in binding + for entrypoint in ("plan", "workspace_size", "ragged_run"): + assert (f"TVM_FFI_DLL_EXPORT_TYPED_FUNC({entrypoint}," in binding) == ( + source_count == 10 ) - ) - assert specs == [sentinel] - assert len(calls) == 1 - assert calls[0]["backend"] == "fa2" - -def test_gen_attention_aot_matrix_contains_equal_primaries_only(tmp_path, monkeypatch): +def test_gen_attention_aot_matrix_contains_equal_primaries_only(generated_sources): import flashinfer.aot as flashinfer_aot - repo_root = Path(__file__).resolve().parents[2] - monkeypatch.setattr(core, "check_cuda_arch", lambda: None) - monkeypatch.setattr( - attention_modules.current_compilation_context, - "TARGET_CUDA_ARCHS", - {(10, 3)}, - ) - monkeypatch.setattr(jit_env, "FLASHINFER_GEN_SRC_DIR", tmp_path / "generated") - monkeypatch.setattr(jit_env, "FLASHINFER_CSRC_DIR", repo_root / "csrc") - specs = list( flashinfer_aot.gen_attention( f16_dtype_=[torch.bfloat16], @@ -386,17 +208,7 @@ def test_gen_attention_aot_matrix_contains_equal_primaries_only(tmp_path, monkey ) -def test_attention_sink_stays_independent_full(tmp_path, monkeypatch): - repo_root = Path(__file__).resolve().parents[2] - monkeypatch.setattr(core, "check_cuda_arch", lambda: None) - monkeypatch.setattr( - attention_modules.current_compilation_context, - "TARGET_CUDA_ARCHS", - {(10, 3)}, - ) - monkeypatch.setattr(jit_env, "FLASHINFER_GEN_SRC_DIR", tmp_path / "generated") - monkeypatch.setattr(jit_env, "FLASHINFER_CSRC_DIR", repo_root / "csrc") - +def test_attention_sink_stays_independent_full(generated_sources): args = ( "fa2", torch.bfloat16, @@ -412,17 +224,18 @@ def test_attention_sink_stays_independent_full(tmp_path, monkeypatch): assert spec.name == attention_modules.get_batch_prefill_attention_sink_uri(*args) assert len(spec.sources) == 10 - generated_dir = tmp_path / "generated" / spec.name + generated_dir = generated_sources / spec.name config = (generated_dir / "batch_prefill_config.inc").read_text() assert "#define PAGED_KV_STRIDE_MODE PAGED_KV_STRIDE_MODE_INDEPENDENT" in config binding = (generated_dir / "batch_prefill_jit_binding.cu").read_text() - assert binding.count("TVM_FFI_DLL_EXPORT_TYPED_FUNC(") == 4 + for entrypoint in ("plan", "workspace_size", "ragged_run", "paged_run"): + assert f"TVM_FFI_DLL_EXPORT_TYPED_FUNC({entrypoint}," in binding for mask_mode in range(4): paged_text = ( generated_dir / f"batch_prefill_paged_kernel_mask_{mask_mode}.cu" ).read_text() - assert paged_text.count("/*SAME_KV_STRIDES=*/true") == 0 - assert paged_text.count("/*SAME_KV_STRIDES=*/false") == 3 + assert "/*SAME_KV_STRIDES=*/true" not in paged_text + assert "/*SAME_KV_STRIDES=*/false" in paged_text def test_batch_prefill_nvfp4_requires_sf_tensors(): diff --git a/tests/jit/test_batch_prefill_lazy_loader.py b/tests/jit/test_batch_prefill_lazy_loader.py index 771de8a8b8a..cedbc422d2d 100644 --- a/tests/jit/test_batch_prefill_lazy_loader.py +++ b/tests/jit/test_batch_prefill_lazy_loader.py @@ -59,7 +59,7 @@ def paged_run(*args): ) -def _run_paged(module, paged_k_cache, paged_v_cache): +def _run_paged(module, paged_k_cache, paged_v_cache, **kwargs): tensor = torch.empty(1) return module.paged_run( tensor, @@ -93,6 +93,7 @@ def _run_paged(module, paged_k_cache, paged_v_cache): 1.0, 0, 0, + **kwargs, ) @@ -161,23 +162,10 @@ def test_concurrent_first_use_loads_once(monkeypatch): assert spec.build_count == 1 -def test_fa2_routes_complete_data_strides_under_one_logical_operation(monkeypatch): +def test_fa2_selects_stride_variant_once(monkeypatch): calls = [] primary_spec = _FakeSpec(_raw_module("primary", calls)) independent_spec = _FakeSpec(_raw_module("independent", calls)) - registered_ops = [] - - def record_custom_op(name, *args, **kwargs): - def decorator(func): - registered_ops.append(name) - return func - - return decorator - - monkeypatch.setattr(prefill, "register_custom_op", record_custom_op) - monkeypatch.setattr( - prefill, "register_fake_op", lambda *args, **kwargs: lambda f: f - ) monkeypatch.setattr( prefill, "_gen_batch_prefill_primary_module", lambda *args: primary_spec ) @@ -189,11 +177,6 @@ def decorator(func): monkeypatch.setattr(torch.cuda, "is_current_stream_capturing", lambda: False) module = prefill.get_batch_prefill_module("fa2", *_MODULE_ARGS) - logical_uri = prefill.get_batch_prefill_uri("fa2", *_MODULE_ARGS) - assert registered_ops == [ - f"flashinfer::{logical_uri}_ragged_run", - f"flashinfer::{logical_uri}_paged_run", - ] assert primary_spec.build_count == 1 assert independent_spec.build_count == 0 @@ -201,7 +184,16 @@ def decorator(func): equal_k = torch.empty_strided(shape, (60, 20, 5, 1)) equal_v = torch.empty_strided(shape, (60, 20, 5, 1)) unequal_v = torch.empty_strided(shape, (80, 20, 5, 1)) - _run_paged(module, equal_k, equal_v) + # Scale-factor layouts must not trigger the independent data-stride module. + scales = dict( + key_block_scales=torch.empty_strided(shape, (60, 20, 5, 1), dtype=torch.uint8), + value_block_scales=torch.empty_strided( + shape, (80, 20, 5, 1), dtype=torch.uint8 + ), + ) + _run_paged(module, equal_k, equal_v, **scales) + assert calls == ["primary"] + assert independent_spec.build_count == 0 _run_paged(module, equal_k, unequal_v) _run_paged(module, equal_k, equal_v) @@ -251,42 +243,24 @@ def prewarm_paged_kv_stride_variant(self, variant): self.variants.append(variant) -def test_standard_wrappers_expose_post_plan_prewarm(): - cached_modules = [] - - prefill_wrapper = prefill.BatchPrefillWithPagedKVCacheWrapper.__new__( - prefill.BatchPrefillWithPagedKVCacheWrapper - ) - prefill_wrapper._plan_info = [1] - prefill_wrapper._cached_module = _PrewarmRecorder() - prefill_wrapper._jit_module = None - prefill_wrapper._backend = "fa2" - cached_modules.append(prefill_wrapper._cached_module) - - decode_wrapper = BatchDecodeWithPagedKVCacheWrapper.__new__( - BatchDecodeWithPagedKVCacheWrapper - ) - decode_wrapper._plan_info = [1] - decode_wrapper._cached_module = _PrewarmRecorder() - decode_wrapper._jit_module = None - decode_wrapper._backend = "fa2" - decode_wrapper._use_tensor_cores = True - cached_modules.append(decode_wrapper._cached_module) - - for wrapper in (prefill_wrapper, decode_wrapper): - wrapper.prewarm_paged_kv_stride_variant("independent") - - assert all(module.variants == ["independent"] for module in cached_modules) - - unplanned = prefill.BatchPrefillWithPagedKVCacheWrapper.__new__( - prefill.BatchPrefillWithPagedKVCacheWrapper - ) - unplanned._plan_info = None - unplanned._cached_module = None - unplanned._jit_module = None - unplanned._backend = "fa2" +@pytest.mark.parametrize( + "wrapper_cls", + [prefill.BatchPrefillWithPagedKVCacheWrapper, BatchDecodeWithPagedKVCacheWrapper], +) +def test_standard_wrappers_expose_post_plan_prewarm(wrapper_cls): + wrapper = wrapper_cls.__new__(wrapper_cls) + wrapper._plan_info = [1] + wrapper._cached_module = _PrewarmRecorder() + wrapper._jit_module = None + wrapper._backend = "fa2" + wrapper._use_tensor_cores = True + + wrapper.prewarm_paged_kv_stride_variant("independent") + assert wrapper._cached_module.variants == ["independent"] + + wrapper._plan_info = None with pytest.raises(RuntimeError, match=r"plan\(\) must complete"): - unplanned.prewarm_paged_kv_stride_variant() + wrapper.prewarm_paged_kv_stride_variant() def test_cascade_wrappers_delegate_prewarm(): diff --git a/tests/test_helpers/paged_kv.py b/tests/test_helpers/paged_kv.py index 91998980233..37bf4f3d01d 100644 --- a/tests/test_helpers/paged_kv.py +++ b/tests/test_helpers/paged_kv.py @@ -16,58 +16,22 @@ from __future__ import annotations -from collections.abc import Callable -from typing import Any - import torch -def make_padded_view( - source: torch.Tensor, - dimension: int, - *, - padding: int = 2, -) -> torch.Tensor: - """Copy ``source`` into an equal-shaped view with padding in one dimension.""" - if padding <= 0: - raise ValueError(f"padding must be positive, got {padding}") - dimension %= source.ndim - parent_shape = list(source.shape) - parent_shape[dimension] += padding - parent = torch.empty( - parent_shape, - dtype=source.dtype, - device=source.device, - ) - view = parent.narrow(dimension, 0, source.shape[dimension]) - view.copy_(source) - if view.stride() == source.stride(): - raise AssertionError( - f"padding dimension {dimension} did not change strides for {source.shape}" - ) - return view - - def make_padded_paged_kv_view( source: torch.Tensor, kv_layout: str, *, padding_heads: int = 2, ) -> torch.Tensor: - """Return an equal-shaped paged-KV view with a padded head dimension.""" - if source.ndim != 4: - raise ValueError(f"expected rank-4 paged KV, got shape {source.shape}") - if kv_layout == "NHD": - head_dimension = 2 - elif kv_layout == "HND": - head_dimension = 1 - else: - raise ValueError(f"kv_layout must be 'NHD' or 'HND', got {kv_layout!r}") - return make_padded_view(source, head_dimension, padding=padding_heads) - - -def get_closure_value(function: Callable[..., Any], name: str) -> Any: - """Return a named closure value from a routed test-facing callable.""" - closure = function.__closure__ or () - values = dict(zip(function.__code__.co_freevars, closure, strict=True)) - return values[name].cell_contents + """Copy paged KV into an equal-shaped view with different outer strides.""" + head_dimension = {"NHD": 2, "HND": 1}[kv_layout] + shape = list(source.shape) + shape[head_dimension] += padding_heads + view = torch.empty(shape, dtype=source.dtype, device=source.device).narrow( + head_dimension, 0, source.shape[head_dimension] + ) + view.copy_(source) + assert view.stride() != source.stride() + return view diff --git a/tests/trace/test_batch_pod_run_reference_correctness.py b/tests/trace/test_batch_pod_run_reference_correctness.py index 268de165d37..9f30d49f731 100644 --- a/tests/trace/test_batch_pod_run_reference_correctness.py +++ b/tests/trace/test_batch_pod_run_reference_correctness.py @@ -6,7 +6,6 @@ from tests.trace.reference_utils import ( _check, ) -from tests.test_helpers.paged_kv import make_padded_paged_kv_view @pytest.mark.parametrize( @@ -87,90 +86,3 @@ def test_batch_pod_run_reference_correctness(shape_kwargs): _check(batch_pod_with_paged_kv_cache_run_trace, ref_d, out_d, atol=1e-3, rtol=1e-3) if torch.cuda.is_available(): torch.cuda.synchronize() - - -def test_batch_pod_independent_paged_kv_strides(): - """Reuse one fused BatchPOD module across unequal prefill and decode KV.""" - from flashinfer.pod import BatchPODWithPagedKVCacheWrapper, get_batch_pod_module - from flashinfer.trace.templates.attention import ( - batch_pod_with_paged_kv_cache_run_trace, - ) - - inputs = batch_pod_with_paged_kv_cache_run_trace.init( - device="cuda", - prefill_len=8, - decode_batch_size=1, - num_pages=2, - num_qo_heads=8, - num_kv_heads=2, - head_dim=64, - page_size=8, - ) - plan = inputs["plan"] - run = inputs["run"] - k_cache_p, v_cache_p = run["paged_kv_cache_p"] - k_cache_d, v_cache_d = run["paged_kv_cache_d"] - v_cache_p_unequal = make_padded_paged_kv_view(v_cache_p, "NHD") - v_cache_d_unequal = make_padded_paged_kv_view(v_cache_d, "NHD") - wrapper = BatchPODWithPagedKVCacheWrapper( - torch.empty(128 * 1024 * 1024, dtype=torch.uint8, device="cuda"), "NHD" - ) - wrapper.plan( - plan["qo_indptr_p"], - plan["kv_indptr_p"], - plan["kv_indices_p"], - plan["last_page_len_p"], - plan["qo_indptr_d"], - plan["kv_indptr_d"], - plan["kv_indices_d"], - plan["last_page_len_d"], - plan["num_qo_heads"], - plan["num_kv_heads"], - plan["head_dim"], - plan["page_size"], - q_data_type=plan["q_data_type"], - kv_data_type=plan["kv_data_type"], - ) - plan_info_p = tuple(wrapper._plan_info_p) - plan_info_d = tuple(wrapper._plan_info_d) - planning_module = wrapper._cached_module - equal_output = wrapper.run( - run["q_p"], - (k_cache_p, v_cache_p), - run["q_d"], - (k_cache_d, v_cache_d), - causal_p=False, - ) - fused_module_misses = get_batch_pod_module.cache_info().misses - unequal_output = wrapper.run( - run["q_p"], - (k_cache_p, v_cache_p_unequal), - run["q_d"], - (k_cache_d, v_cache_d_unequal), - causal_p=False, - ) - assert get_batch_pod_module.cache_info().misses == fused_module_misses - assert wrapper._cached_module is planning_module - assert tuple(wrapper._plan_info_p) == plan_info_p - assert tuple(wrapper._plan_info_d) == plan_info_d - - expected = batch_pod_with_paged_kv_cache_run_trace.reference( - run["q_p"], - (k_cache_p, v_cache_p), - run["q_d"], - (k_cache_d, v_cache_d), - ) - _check( - batch_pod_with_paged_kv_cache_run_trace, - expected, - equal_output, - atol=1e-3, - rtol=1e-3, - ) - _check( - batch_pod_with_paged_kv_cache_run_trace, - expected, - unequal_output, - atol=1e-3, - rtol=1e-3, - ) diff --git a/tests/trace/test_pod_with_paged_kv_cache_run_reference_correctness.py b/tests/trace/test_pod_with_paged_kv_cache_run_reference_correctness.py index 04c427b2682..1fa625fc1b2 100644 --- a/tests/trace/test_pod_with_paged_kv_cache_run_reference_correctness.py +++ b/tests/trace/test_pod_with_paged_kv_cache_run_reference_correctness.py @@ -6,7 +6,6 @@ from tests.trace.reference_utils import ( _check, ) -from tests.test_helpers.paged_kv import make_padded_paged_kv_view @pytest.mark.parametrize( @@ -87,82 +86,3 @@ def test_pod_with_paged_kv_cache_run_reference_correctness(shape_kwargs): ) if torch.cuda.is_available(): torch.cuda.synchronize() - - -def test_pod_independent_paged_kv_strides(): - """Reuse one POD plan and fused module for equal and unequal decode KV.""" - from flashinfer.pod import PODWithPagedKVCacheWrapper, get_pod_module - from flashinfer.trace.templates.attention import pod_with_paged_kv_cache_run_trace - - inputs = pod_with_paged_kv_cache_run_trace.init( - device="cuda", - prefill_len=8, - decode_batch_size=1, - num_pages=2, - num_qo_heads=8, - num_kv_heads=2, - head_dim=64, - page_size=8, - ) - plan = inputs["plan"] - run = inputs["run"] - k_cache, v_cache = run["paged_kv_cache_d"] - v_cache_unequal = make_padded_paged_kv_view(v_cache, "NHD") - wrapper = PODWithPagedKVCacheWrapper( - torch.empty(64 * 1024 * 1024, dtype=torch.uint8, device="cuda"), "NHD" - ) - wrapper.plan( - plan["kv_indptr"], - plan["kv_indices"], - plan["kv_last_page_len"], - plan["num_qo_heads"], - plan["num_kv_heads"], - plan["head_dim"], - plan["page_size"], - q_data_type=plan["q_data_type"], - kv_data_type=plan["kv_data_type"], - ) - plan_info = tuple(wrapper._plan_info) - planning_module = wrapper._cached_module - equal_output = wrapper.run( - run["q_p"], - run["k_p"], - run["v_p"], - run["q_d"], - (k_cache, v_cache), - causal_p=True, - ) - fused_module_misses = get_pod_module.cache_info().misses - unequal_output = wrapper.run( - run["q_p"], - run["k_p"], - run["v_p"], - run["q_d"], - (k_cache, v_cache_unequal), - causal_p=True, - ) - assert get_pod_module.cache_info().misses == fused_module_misses - assert wrapper._cached_module is planning_module - assert tuple(wrapper._plan_info) == plan_info - - expected = pod_with_paged_kv_cache_run_trace.reference( - run["q_p"], - run["k_p"], - run["v_p"], - run["q_d"], - (k_cache, v_cache), - ) - _check( - pod_with_paged_kv_cache_run_trace, - expected, - equal_output, - atol=1e-3, - rtol=1e-3, - ) - _check( - pod_with_paged_kv_cache_run_trace, - expected, - unequal_output, - atol=1e-3, - rtol=1e-3, - ) diff --git a/tests/trace/test_var_block_sparse_run_reference_correctness.py b/tests/trace/test_var_block_sparse_run_reference_correctness.py index 7504ff0ddb7..480099002a1 100644 --- a/tests/trace/test_var_block_sparse_run_reference_correctness.py +++ b/tests/trace/test_var_block_sparse_run_reference_correctness.py @@ -6,7 +6,6 @@ from tests.trace.reference_utils import ( _check, ) -from tests.test_helpers.paged_kv import get_closure_value, make_padded_view @pytest.mark.parametrize( @@ -81,84 +80,3 @@ def test_var_block_sparse_run_reference_correctness(shape_kwargs): ) if torch.cuda.is_available(): torch.cuda.synchronize() - - -def test_var_block_sparse_lazy_stride_router(monkeypatch): - """Normalize unequal HND inputs onto the shared routed FA2 namespace.""" - from flashinfer import VariableBlockSparseAttentionWrapper - from flashinfer.trace.templates.attention import ( - variable_block_sparse_attention_run_trace, - ) - - inputs = variable_block_sparse_attention_run_trace.init( - device="cuda", - qo_len=32, - kv_len=32, - num_qo_heads=8, - num_kv_heads=2, - head_dim=64, - ) - R = C = 16 - M, num_qo_heads, head_dim = inputs["q"].shape - N, num_kv_heads, _ = inputs["k"].shape - block_mask_map = torch.ones( - num_kv_heads, M // R, N // C, dtype=torch.bool, device="cuda" - ) - block_row_sz = torch.full( - (num_kv_heads, M // R), R, dtype=torch.int32, device="cuda" - ) - block_col_sz = torch.full( - (num_kv_heads, N // C), C, dtype=torch.int32, device="cuda" - ) - q_hnd = inputs["q"].transpose(0, 1).contiguous() - k_hnd = inputs["k"].transpose(0, 1).contiguous() - v_hnd = inputs["v"].transpose(0, 1).contiguous() - v_hnd_unequal = make_padded_view(v_hnd, 1) - assert k_hnd.stride() == v_hnd.stride() - assert k_hnd.stride() != v_hnd_unequal.stride() - - wrapper = VariableBlockSparseAttentionWrapper( - torch.empty(128 * 1024 * 1024, device="cuda"), backend="fa2" - ) - wrapper.plan( - block_mask_map=block_mask_map, - block_row_sz=block_row_sz, - block_col_sz=block_col_sz, - num_qo_heads=num_qo_heads, - num_kv_heads=num_kv_heads, - head_dim=head_dim, - q_data_type=torch.float16, - kv_data_type=torch.float16, - ) - plan_info = tuple(wrapper._plan_info) - routed_module = wrapper._cached_module - holder = get_closure_value(routed_module.paged_run, "lazy_independent_module") - - def unexpected_supplement_load(): - raise AssertionError( - "variable-block sparse must normalize HND tensors before paged routing" - ) - - monkeypatch.setattr(holder, "get", unexpected_supplement_load) - equal_output = wrapper.run(q_hnd, k_hnd, v_hnd) - unequal_input_output = wrapper.run(q_hnd, k_hnd, v_hnd_unequal) - assert wrapper._cached_module is routed_module - assert tuple(wrapper._plan_info) == plan_info - - expected = variable_block_sparse_attention_run_trace.reference( - inputs["q"], inputs["k"], inputs["v"] - ) - _check( - variable_block_sparse_attention_run_trace, - expected, - equal_output.transpose(0, 1), - atol=1e-2, - rtol=1e-2, - ) - _check( - variable_block_sparse_attention_run_trace, - expected, - unequal_input_output.transpose(0, 1), - atol=1e-2, - rtol=1e-2, - )