diff --git a/csrc/batch_pod.cu b/csrc/batch_pod.cu index 98ff9d83dae..8114d9b9eaf 100644 --- a/csrc/batch_pod.cu +++ b/csrc/batch_pod.cu @@ -90,15 +90,10 @@ void batch_pod_with_kv_cache_tensor( const auto q_stride_n_p = q_p.stride(0); const auto q_stride_h_p = q_p.stride(1); - // get kv_cache_strides - const int64_t* kv_cache_strides_p = nullptr; + // get independent K/V cache strides auto k_strides_p = paged_k_cache_p.strides(); auto v_strides_p = paged_v_cache_p.strides(); TVM_FFI_ICHECK_EQ(k_strides_p.size(), v_strides_p.size()); - for (int i = 0; i < k_strides_p.size(); ++i) { - TVM_FFI_ICHECK_EQ(k_strides_p[i], v_strides_p[i]); - } - kv_cache_strides_p = k_strides_p.data(); ffi::CUDADeviceGuard device_guard(float_workspace_buffer_p.device().device_id); const cudaStream_t stream = get_stream(float_workspace_buffer_p.device()); @@ -141,15 +136,10 @@ void batch_pod_with_kv_cache_tensor( const auto q_stride_n_d = q_d.stride(0); const auto q_stride_h_d = q_d.stride(1); - // get kv_cache_strides - const int64_t* kv_cache_strides_d = nullptr; + // get independent K/V cache strides auto k_strides_d = paged_k_cache_d.strides(); auto v_strides_d = paged_v_cache_d.strides(); TVM_FFI_ICHECK_EQ(k_strides_d.size(), v_strides_d.size()); - for (int i = 0; i < k_strides_d.size(); ++i) { - TVM_FFI_ICHECK_EQ(k_strides_d[i], v_strides_d[i]); - } - kv_cache_strides_d = k_strides_d.data(); // Already handled by prefill // ffi::CUDADeviceGuard device_guard(float_workspace_buffer_d.device().device_id); @@ -167,8 +157,8 @@ void batch_pod_with_kv_cache_tensor( paged_kv_t paged_kv( num_kv_heads_p, page_size_p, HEAD_DIM_VO, batch_size_p, kv_layout_p, static_cast(paged_k_cache_p.data_ptr()), - static_cast(paged_v_cache_p.data_ptr()), kv_cache_strides_p, - static_cast(paged_kv_indices_p.data_ptr()), + static_cast(paged_v_cache_p.data_ptr()), k_strides_p.data(), + v_strides_p.data(), static_cast(paged_kv_indices_p.data_ptr()), static_cast(paged_kv_indptr_p.data_ptr()), static_cast(paged_kv_last_page_len_p.data_ptr())); params.paged_kv = paged_kv; @@ -245,8 +235,8 @@ void batch_pod_with_kv_cache_tensor( paged_kv_t paged_kv( num_kv_heads_d, page_size_d, HEAD_DIM_VO, batch_size_d, kv_layout_d, static_cast(paged_k_cache_d.data_ptr()), - static_cast(paged_v_cache_d.data_ptr()), kv_cache_strides_d, - static_cast(paged_kv_indices_d.data_ptr()), + static_cast(paged_v_cache_d.data_ptr()), k_strides_d.data(), + v_strides_d.data(), static_cast(paged_kv_indices_d.data_ptr()), static_cast(paged_kv_indptr_d.data_ptr()), static_cast(paged_kv_last_page_len_d.data_ptr())); params.paged_kv = paged_kv; diff --git a/csrc/batch_prefill.cu b/csrc/batch_prefill.cu index 1f083268452..a7cfaaf46bb 100644 --- a/csrc/batch_prefill.cu +++ b/csrc/batch_prefill.cu @@ -23,7 +23,7 @@ namespace flashinfer { -template cudaError_t BatchPrefillWithPagedKVCacheDispatched(Params params, typename Params::DTypeO* tmp_v, @@ -236,138 +236,4 @@ void BatchPrefillWithRaggedKVCacheRun(TensorView float_workspace_buffer, }); } -void BatchPrefillWithPagedKVCacheRun(TensorView float_workspace_buffer, - TensorView int_workspace_buffer, Array plan_info_vec, - TensorView q, TensorView paged_k_cache, - TensorView paged_v_cache, TensorView qo_indptr, - TensorView paged_kv_indptr, TensorView paged_kv_indices, - TensorView paged_kv_last_page_len, TensorView o, - Optional maybe_lse, int64_t mask_mode_code, - int64_t layout, int64_t window_left, - bool enable_pdl ADDITIONAL_FUNC_PARAMS) { - PrefillPlanInfo plan_info; - plan_info.FromVector(std::vector(plan_info_vec.begin(), plan_info_vec.end())); - QKVLayout kv_layout = static_cast(layout); - int64_t batch_size = paged_kv_indptr.size(0) - 1; - int64_t num_qo_heads = q.size(1); - int64_t num_kv_heads, page_size; - uint32_t head_dim_qk = q.size(2); - if (kv_layout == QKVLayout::kHND) { - num_kv_heads = paged_k_cache.size(1); - page_size = paged_k_cache.size(2); - } else { - page_size = paged_k_cache.size(1); - num_kv_heads = paged_k_cache.size(2); - } - - if (maybe_lse) { - const auto& lse = *maybe_lse; - TVM_FFI_ICHECK_EQ(lse.size(0), q.size(0)); - TVM_FFI_ICHECK_EQ(lse.size(1), q.size(1)); - } - - void* float_buffer_ptr = static_cast(float_workspace_buffer.data_ptr()); - void* int_buffer_ptr = static_cast(int_workspace_buffer.data_ptr()); - - const MaskMode mask_mode = static_cast(mask_mode_code); - - // get q_stride_n and q_stride_h - const auto q_stride_n = q.stride(0); - const auto q_stride_h = q.stride(1); - - // get kv-cache strides - auto k_cache_strides = paged_k_cache.strides(); - auto v_cache_strides = paged_v_cache.strides(); - TVM_FFI_ICHECK_EQ(paged_k_cache.ndim(), paged_v_cache.ndim()); - - ffi::CUDADeviceGuard device_guard(float_workspace_buffer.device().device_id); - const cudaStream_t stream = get_stream(float_workspace_buffer.device()); - - DISPATCH_context( - DTypeQ, DTypeKV, DTypeO, IdType, MASK_MODE, HEAD_DIM_QK, HEAD_DIM_VO, POS_ENCODING_MODE, - USE_SLIDING_WINDOW, USE_LOGITS_SOFT_CAP, USE_FP16_QK_REDUCTION, AttentionVariant, - RaggedParams, PagedParams, [&] { - PagedParams params; - - params.q = static_cast(q.data_ptr()); - paged_kv_t paged_kv( - num_kv_heads, page_size, HEAD_DIM_VO, batch_size, kv_layout, - static_cast(paged_k_cache.data_ptr()), - static_cast(paged_v_cache.data_ptr()), k_cache_strides.data(), - v_cache_strides.data(), static_cast(paged_kv_indices.data_ptr()), - static_cast(paged_kv_indptr.data_ptr()), - static_cast(paged_kv_last_page_len.data_ptr())); - params.paged_kv = paged_kv; - params.q_indptr = static_cast(qo_indptr.data_ptr()); - params.o = static_cast(o.data_ptr()); - - params.lse = maybe_lse ? static_cast(maybe_lse.value().data_ptr()) : nullptr; - params.num_qo_heads = num_qo_heads; - params.group_size = uint_fastdiv(num_qo_heads / paged_kv.num_heads); - params.q_stride_n = q_stride_n; - params.q_stride_h = q_stride_h; - params.window_left = window_left; - - params.request_indices = nullptr; - params.qo_tile_indices = nullptr; - params.kv_tile_indices = nullptr; - params.merge_indptr = nullptr; - params.o_indptr = nullptr; - params.kv_chunk_size_ptr = nullptr; - params.block_valid_mask = nullptr; - params.total_num_rows = nullptr; - params.max_total_num_rows = 0; - params.padded_batch_size = 0; - params.partition_kv = false; - params.k_sf_stride_page = 0; - params.k_sf_stride_n = 0; - params.k_sf_stride_h = 0; - params.v_sf_stride_page = 0; - params.v_sf_stride_n = 0; - params.v_sf_stride_h = 0; - - ADDITIONAL_PARAMS_SETTER - - DTypeO* tmp_v = nullptr; - float* tmp_s = nullptr; - - params.request_indices = - GetPtrFromBaseOffset(int_buffer_ptr, plan_info.request_indices_offset); - params.qo_tile_indices = - GetPtrFromBaseOffset(int_buffer_ptr, plan_info.qo_tile_indices_offset); - params.kv_tile_indices = - GetPtrFromBaseOffset(int_buffer_ptr, plan_info.kv_tile_indices_offset); - params.o_indptr = GetPtrFromBaseOffset(int_buffer_ptr, plan_info.o_indptr_offset); - params.kv_chunk_size_ptr = - GetPtrFromBaseOffset(int_buffer_ptr, plan_info.kv_chunk_size_ptr_offset); - if (plan_info.split_kv) { - params.merge_indptr = - GetPtrFromBaseOffset(int_buffer_ptr, plan_info.merge_indptr_offset); - tmp_v = GetPtrFromBaseOffset(float_buffer_ptr, plan_info.v_offset); - tmp_s = GetPtrFromBaseOffset(float_buffer_ptr, plan_info.s_offset); - if (plan_info.enable_cuda_graph) { - params.block_valid_mask = - GetPtrFromBaseOffset(int_buffer_ptr, plan_info.block_valid_mask_offset); - } - } - params.padded_batch_size = plan_info.padded_batch_size; - params.max_total_num_rows = plan_info.total_num_rows; - if (plan_info.enable_cuda_graph) { - params.total_num_rows = - GetPtrFromBaseOffset(int_buffer_ptr, plan_info.total_num_rows_offset); - } - - cudaError_t status = cudaSuccess; - - DISPATCH_CTA_TILE_Q(plan_info.cta_tile_q, CTA_TILE_Q, { - status = flashinfer::BatchPrefillWithPagedKVCacheDispatched< - CTA_TILE_Q, HEAD_DIM_QK, HEAD_DIM_VO, POS_ENCODING_MODE, - /*use_fp16_qk_reduction=*/USE_FP16_QK_REDUCTION, MASK_MODE, AttentionVariant, - PagedParams>(params, tmp_v, tmp_s, enable_pdl, stream); - }); - - TVM_FFI_ICHECK(status == cudaSuccess) - << "BatchPrefillWithPagedKVCache failed with error " << cudaGetErrorString(status); - return true; - }); -} +#include "batch_prefill_paged.cuh" diff --git a/csrc/batch_prefill_customize_config.jinja b/csrc/batch_prefill_customize_config.jinja index 6f5f1d9aa42..c9ee34e41be 100644 --- a/csrc/batch_prefill_customize_config.jinja +++ b/csrc/batch_prefill_customize_config.jinja @@ -38,6 +38,11 @@ constexpr auto USE_LOGITS_SOFT_CAP = {{ use_logits_soft_cap }}; constexpr auto POS_ENCODING_MODE = {{ pos_encoding_mode }}; constexpr auto USE_SLIDING_WINDOW = {{ use_sliding_window }}; +#define PAGED_KV_STRIDE_MODE_RUNTIME 0 +#define PAGED_KV_STRIDE_MODE_EQUAL 1 +#define PAGED_KV_STRIDE_MODE_INDEPENDENT 2 +#define PAGED_KV_STRIDE_MODE PAGED_KV_STRIDE_MODE_{{ paged_kv_stride_mode | upper }} + struct RaggedParams { using DTypeQ = DTypeQ; diff --git a/csrc/batch_prefill_paged.cu b/csrc/batch_prefill_paged.cu new file mode 100644 index 00000000000..e2e8dd9f1c2 --- /dev/null +++ b/csrc/batch_prefill_paged.cu @@ -0,0 +1,40 @@ +/* + * Copyright (c) 2023 by FlashInfer team. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include +#include +#include + +#include "batch_prefill_config.inc" +#include "tvm/ffi/container/array.h" +#include "tvm_ffi_utils.h" + +namespace flashinfer { + +template +cudaError_t BatchPrefillWithPagedKVCacheDispatched(Params params, typename Params::DTypeO* tmp_v, + float* tmp_s, bool enable_pdl, + cudaStream_t stream); + +} // namespace flashinfer + +using namespace flashinfer; + +using tvm::ffi::Array; +using tvm::ffi::Optional; + +#include "batch_prefill_paged.cuh" diff --git a/csrc/batch_prefill_paged.cuh b/csrc/batch_prefill_paged.cuh new file mode 100644 index 00000000000..66995401dda --- /dev/null +++ b/csrc/batch_prefill_paged.cuh @@ -0,0 +1,181 @@ +/* + * Copyright (c) 2023 by FlashInfer team. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#pragma once + +void BatchPrefillWithPagedKVCacheRun(TensorView float_workspace_buffer, + TensorView int_workspace_buffer, Array plan_info_vec, + TensorView q, TensorView paged_k_cache, + TensorView paged_v_cache, TensorView qo_indptr, + TensorView paged_kv_indptr, TensorView paged_kv_indices, + TensorView paged_kv_last_page_len, TensorView o, + Optional maybe_lse, int64_t mask_mode_code, + int64_t layout, int64_t window_left, + bool enable_pdl ADDITIONAL_FUNC_PARAMS) { + PrefillPlanInfo plan_info; + plan_info.FromVector(std::vector(plan_info_vec.begin(), plan_info_vec.end())); + QKVLayout kv_layout = static_cast(layout); + int64_t batch_size = paged_kv_indptr.size(0) - 1; + int64_t num_qo_heads = q.size(1); + int64_t num_kv_heads, page_size; + uint32_t head_dim_qk = q.size(2); + if (kv_layout == QKVLayout::kHND) { + num_kv_heads = paged_k_cache.size(1); + page_size = paged_k_cache.size(2); + } else { + page_size = paged_k_cache.size(1); + num_kv_heads = paged_k_cache.size(2); + } + + if (maybe_lse) { + const auto& lse = *maybe_lse; + TVM_FFI_ICHECK_EQ(lse.size(0), q.size(0)); + TVM_FFI_ICHECK_EQ(lse.size(1), q.size(1)); + } + + void* float_buffer_ptr = static_cast(float_workspace_buffer.data_ptr()); + void* int_buffer_ptr = static_cast(int_workspace_buffer.data_ptr()); + + const MaskMode mask_mode = static_cast(mask_mode_code); + + // get q_stride_n and q_stride_h + const auto q_stride_n = q.stride(0); + const auto q_stride_h = q.stride(1); + + // get kv-cache strides + auto k_cache_strides = paged_k_cache.strides(); + auto v_cache_strides = paged_v_cache.strides(); + TVM_FFI_ICHECK_EQ(paged_k_cache.ndim(), paged_v_cache.ndim()); + bool kv_strides_are_identical = true; + for (int i = 0; i < paged_k_cache.ndim(); ++i) { + kv_strides_are_identical &= paged_k_cache.stride(i) == paged_v_cache.stride(i); + } + + ffi::CUDADeviceGuard device_guard(float_workspace_buffer.device().device_id); + const cudaStream_t stream = get_stream(float_workspace_buffer.device()); + + DISPATCH_context( + DTypeQ, DTypeKV, DTypeO, IdType, MASK_MODE, HEAD_DIM_QK, HEAD_DIM_VO, POS_ENCODING_MODE, + USE_SLIDING_WINDOW, USE_LOGITS_SOFT_CAP, USE_FP16_QK_REDUCTION, AttentionVariant, + RaggedParams, PagedParams, [&] { + PagedParams params; + + params.q = static_cast(q.data_ptr()); + paged_kv_t paged_kv( + num_kv_heads, page_size, HEAD_DIM_VO, batch_size, kv_layout, + static_cast(paged_k_cache.data_ptr()), + static_cast(paged_v_cache.data_ptr()), k_cache_strides.data(), + v_cache_strides.data(), static_cast(paged_kv_indices.data_ptr()), + static_cast(paged_kv_indptr.data_ptr()), + static_cast(paged_kv_last_page_len.data_ptr())); + params.paged_kv = paged_kv; + params.q_indptr = static_cast(qo_indptr.data_ptr()); + params.o = static_cast(o.data_ptr()); + + params.lse = maybe_lse ? static_cast(maybe_lse.value().data_ptr()) : nullptr; + params.num_qo_heads = num_qo_heads; + params.group_size = uint_fastdiv(num_qo_heads / paged_kv.num_heads); + params.q_stride_n = q_stride_n; + params.q_stride_h = q_stride_h; + params.window_left = window_left; + + params.request_indices = nullptr; + params.qo_tile_indices = nullptr; + params.kv_tile_indices = nullptr; + params.merge_indptr = nullptr; + params.o_indptr = nullptr; + params.kv_chunk_size_ptr = nullptr; + params.block_valid_mask = nullptr; + params.total_num_rows = nullptr; + params.max_total_num_rows = 0; + params.padded_batch_size = 0; + params.partition_kv = false; + params.k_sf_stride_page = 0; + params.k_sf_stride_n = 0; + params.k_sf_stride_h = 0; + params.v_sf_stride_page = 0; + params.v_sf_stride_n = 0; + params.v_sf_stride_h = 0; + + ADDITIONAL_PARAMS_SETTER + + DTypeO* tmp_v = nullptr; + float* tmp_s = nullptr; + + params.request_indices = + GetPtrFromBaseOffset(int_buffer_ptr, plan_info.request_indices_offset); + params.qo_tile_indices = + GetPtrFromBaseOffset(int_buffer_ptr, plan_info.qo_tile_indices_offset); + params.kv_tile_indices = + GetPtrFromBaseOffset(int_buffer_ptr, plan_info.kv_tile_indices_offset); + params.o_indptr = GetPtrFromBaseOffset(int_buffer_ptr, plan_info.o_indptr_offset); + params.kv_chunk_size_ptr = + GetPtrFromBaseOffset(int_buffer_ptr, plan_info.kv_chunk_size_ptr_offset); + if (plan_info.split_kv) { + params.merge_indptr = + GetPtrFromBaseOffset(int_buffer_ptr, plan_info.merge_indptr_offset); + tmp_v = GetPtrFromBaseOffset(float_buffer_ptr, plan_info.v_offset); + tmp_s = GetPtrFromBaseOffset(float_buffer_ptr, plan_info.s_offset); + if (plan_info.enable_cuda_graph) { + params.block_valid_mask = + GetPtrFromBaseOffset(int_buffer_ptr, plan_info.block_valid_mask_offset); + } + } + params.padded_batch_size = plan_info.padded_batch_size; + params.max_total_num_rows = plan_info.total_num_rows; + if (plan_info.enable_cuda_graph) { + params.total_num_rows = + GetPtrFromBaseOffset(int_buffer_ptr, plan_info.total_num_rows_offset); + } + + cudaError_t status = cudaSuccess; + +#if PAGED_KV_STRIDE_MODE == PAGED_KV_STRIDE_MODE_RUNTIME + DISPATCH_BOOL(kv_strides_are_identical, SAME_KV_STRIDES, [&] { + DISPATCH_CTA_TILE_Q(plan_info.cta_tile_q, CTA_TILE_Q, { + status = flashinfer::BatchPrefillWithPagedKVCacheDispatched< + /*SAME_KV_STRIDES=*/SAME_KV_STRIDES, CTA_TILE_Q, HEAD_DIM_QK, HEAD_DIM_VO, + POS_ENCODING_MODE, /*use_fp16_qk_reduction=*/USE_FP16_QK_REDUCTION, MASK_MODE, + AttentionVariant, PagedParams>(params, tmp_v, tmp_s, enable_pdl, stream); + }); + return true; + }); +#elif PAGED_KV_STRIDE_MODE == PAGED_KV_STRIDE_MODE_EQUAL + TVM_FFI_ICHECK(kv_strides_are_identical) + << "The equal-stride FA2 batch-prefill primary requires paged K/V cache data " + "tensors to have identical strides in every dimension; route unequal layouts " + "through the independent paged module."; + DISPATCH_CTA_TILE_Q(plan_info.cta_tile_q, CTA_TILE_Q, { + status = flashinfer::BatchPrefillWithPagedKVCacheDispatched< + /*SAME_KV_STRIDES=*/true, CTA_TILE_Q, HEAD_DIM_QK, HEAD_DIM_VO, POS_ENCODING_MODE, + /*use_fp16_qk_reduction=*/USE_FP16_QK_REDUCTION, MASK_MODE, AttentionVariant, + PagedParams>(params, tmp_v, tmp_s, enable_pdl, stream); + }); +#elif PAGED_KV_STRIDE_MODE == PAGED_KV_STRIDE_MODE_INDEPENDENT + DISPATCH_CTA_TILE_Q(plan_info.cta_tile_q, CTA_TILE_Q, { + status = flashinfer::BatchPrefillWithPagedKVCacheDispatched< + /*SAME_KV_STRIDES=*/false, CTA_TILE_Q, HEAD_DIM_QK, HEAD_DIM_VO, POS_ENCODING_MODE, + /*use_fp16_qk_reduction=*/USE_FP16_QK_REDUCTION, MASK_MODE, AttentionVariant, + PagedParams>(params, tmp_v, tmp_s, enable_pdl, stream); + }); +#else +#error "Unsupported PAGED_KV_STRIDE_MODE" +#endif + + TVM_FFI_ICHECK(status == cudaSuccess) + << "BatchPrefillWithPagedKVCache failed with error " << cudaGetErrorString(status); + return true; + }); +} diff --git a/csrc/batch_prefill_paged_jit_binding.cu b/csrc/batch_prefill_paged_jit_binding.cu new file mode 100644 index 00000000000..e1392547339 --- /dev/null +++ b/csrc/batch_prefill_paged_jit_binding.cu @@ -0,0 +1,32 @@ +/* + * Copyright (c) 2023-2025 by FlashInfer team. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "batch_prefill_config.inc" +#include "tvm_ffi_utils.h" + +using tvm::ffi::Array; +using tvm::ffi::Optional; + +void BatchPrefillWithPagedKVCacheRun(TensorView float_workspace_buffer, + TensorView int_workspace_buffer, Array plan_info_vec, + TensorView q, TensorView paged_k_cache, + TensorView paged_v_cache, TensorView qo_indptr, + TensorView paged_kv_indptr, TensorView paged_kv_indices, + TensorView paged_kv_last_page_len, TensorView o, + Optional maybe_lse, int64_t mask_mode_code, + int64_t layout, int64_t window_left, + bool enable_pdl ADDITIONAL_FUNC_PARAMS); + +TVM_FFI_DLL_EXPORT_TYPED_FUNC(paged_run, BatchPrefillWithPagedKVCacheRun); diff --git a/csrc/batch_prefill_paged_kernel_inst.jinja b/csrc/batch_prefill_paged_kernel_inst.jinja index 5518d8b71b5..93a522b7bd0 100644 --- a/csrc/batch_prefill_paged_kernel_inst.jinja +++ b/csrc/batch_prefill_paged_kernel_inst.jinja @@ -9,10 +9,13 @@ constexpr auto use_custom_mask = {{ mask_mode }} == MaskMode::kCustom; {16} for head_dim_qk >= 512 with a smaller head_dim_vo (o_frag register pressure permits only CTA16 there), and {16, 64, 128} otherwise; don't instantiate unreachable variants. #} +{% for same_kv_strides in same_kv_strides_values %} {% for cta_tile_q in ([16, 32] if head_dim_vo | int >= 512 else ([16] if head_dim_qk | int >= 512 else [16, 64, 128])) %} template cudaError_t BatchPrefillWithPagedKVCacheDispatched< - /*CTA_TILE_Q=*/{{cta_tile_q}}, {{head_dim_qk}}, {{head_dim_vo}}, {{pos_encoding_mode}}, {{use_fp16_qk_reduction}}, {{mask_mode}}, + /*SAME_KV_STRIDES=*/{{same_kv_strides}}, /*CTA_TILE_Q=*/{{cta_tile_q}}, + {{head_dim_qk}}, {{head_dim_vo}}, {{pos_encoding_mode}}, {{use_fp16_qk_reduction}}, {{mask_mode}}, {{ variant_name }}, PagedParams>(PagedParams params, {{ dtype_o }}* tmp_v, float* tmp_s, bool enable_pdl, cudaStream_t stream); {% endfor %} +{% endfor %} }; // namespace flashinfer diff --git a/csrc/pod.cu b/csrc/pod.cu index c2a271b0969..24a5ac7fc7e 100644 --- a/csrc/pod.cu +++ b/csrc/pod.cu @@ -123,15 +123,10 @@ void pod_with_kv_cache_tensor( const auto q_stride_n_d = q_d.stride(0); const auto q_stride_h_d = q_d.stride(1); - // get kv_cache_strides - const int64_t* kv_cache_strides_d = nullptr; + // get independent K/V cache strides auto k_strides_d = paged_k_cache_d.strides(); auto v_strides_d = paged_v_cache_d.strides(); TVM_FFI_ICHECK_EQ(k_strides_d.size(), v_strides_d.size()); - for (int i = 0; i < k_strides_d.size(); ++i) { - TVM_FFI_ICHECK_EQ(k_strides_d[i], v_strides_d[i]); - } - kv_cache_strides_d = k_strides_d.data(); ffi::CUDADeviceGuard device_guard(float_workspace_buffer_d.device().device_id); const cudaStream_t stream = get_stream(float_workspace_buffer_d.device()); @@ -187,8 +182,8 @@ void pod_with_kv_cache_tensor( paged_kv_t paged_kv( num_kv_heads, page_size_d, HEAD_DIM_VO, batch_size, kv_layout_d, static_cast(paged_k_cache_d.data_ptr()), - static_cast(paged_v_cache_d.data_ptr()), kv_cache_strides_d, - static_cast(paged_kv_indices_d.data_ptr()), + static_cast(paged_v_cache_d.data_ptr()), k_strides_d.data(), + v_strides_d.data(), static_cast(paged_kv_indices_d.data_ptr()), static_cast(paged_kv_indptr_d.data_ptr()), static_cast(paged_kv_last_page_len_d.data_ptr())); params.paged_kv = paged_kv; diff --git a/flashinfer/aot.py b/flashinfer/aot.py index eeee2e97f74..35b5f3a8f50 100644 --- a/flashinfer/aot.py +++ b/flashinfer/aot.py @@ -44,6 +44,7 @@ gen_trtllm_gen_fmha_module, gen_trtllm_fmha_v2_sm120_module, ) +from .jit.attention.modules import _gen_batch_prefill_primary_module from .jit.attention.utils import _is_nvfp4_kv_dtype from .jit.cascade import gen_cascade_module from .jit.cake_fmha import gen_cake_fmha_compat_module @@ -186,7 +187,7 @@ def gen_fa2( if dtype_qo.itemsize == 1: return # fp8 tensor cores not supported in fa2 - yield gen_batch_prefill_module( + yield _gen_batch_prefill_primary_module( backend="fa2", dtype_q=dtype_qo, dtype_kv=dtype_kv, diff --git a/flashinfer/attention/_core.py b/flashinfer/attention/_core.py index d7d63dddd16..e8bc6117515 100644 --- a/flashinfer/attention/_core.py +++ b/flashinfer/attention/_core.py @@ -390,6 +390,8 @@ def __init__( "use_sliding_window": window_left >= 0, "use_fp16_qk_reduction": use_fp16_qk_reduction, "pos_encoding_mode": PosEncodingMode[pos_encoding_mode].value, + "paged_kv_stride_mode": ("independent" if backend == "fa2" else "runtime"), + "module_surface": "full", } super().__init__( diff --git a/flashinfer/cascade.py b/flashinfer/cascade.py index 67ae92e9f68..97e2f316279 100644 --- a/flashinfer/cascade.py +++ b/flashinfer/cascade.py @@ -516,6 +516,21 @@ def plan( kv_data_type=kv_data_type, ) + @flashinfer_api + def prewarm_paged_kv_stride_variant(self, variant: str = "independent") -> None: + r"""Prewarm every level's lazy paged-KV-stride variant after :meth:`plan`. + + Call this method before CUDA graph capture. + + Parameters + ---------- + variant : str + The paged-KV-stride variant to prewarm. The only supported value is + ``"independent"`` (the default), for K and V with different data strides. + """ + for wrapper in self._batch_prefill_wrappers: + wrapper.prewarm_paged_kv_stride_variant(variant) + begin_forward = plan @flashinfer_api(trace=multi_level_cascade_run_trace) @@ -1002,6 +1017,20 @@ def begin_forward( page_size, ) + @flashinfer_api + def prewarm_paged_kv_stride_variant(self, variant: str = "independent") -> None: + r"""Prewarm the underlying prefill wrapper's paged-KV-stride variant. + + Call this method after :meth:`begin_forward` and before CUDA graph capture. + + Parameters + ---------- + variant : str + The paged-KV-stride variant to prewarm. The only supported value is + ``"independent"`` (the default), for K and V with different data strides. + """ + self._batch_prefill_wrapper.prewarm_paged_kv_stride_variant(variant) + @flashinfer_api def forward( self, diff --git a/flashinfer/decode.py b/flashinfer/decode.py index af3492b0a88..d9e0513843d 100644 --- a/flashinfer/decode.py +++ b/flashinfer/decode.py @@ -2045,6 +2045,40 @@ def _plan_impl( self._q_len_per_req = q_len_per_req self._is_causal = is_causal + @flashinfer_api + def prewarm_paged_kv_stride_variant(self, variant: str = "independent") -> None: + r"""Load a lazy tensor-core paged-KV-stride variant after :meth:`plan`. + + Call this method before CUDA graph capture when a planned standard FA2 + tensor-core decode wrapper will receive K and V tensors with different + data strides. + + Parameters + ---------- + variant : str + The paged-KV-stride variant to prewarm. The only supported value is + ``"independent"`` (the default), for K and V with different data strides. + """ + if ( + getattr(self, "_plan_info", None) is None + or getattr(self, "_cached_module", None) is None + ): + raise RuntimeError( + "plan() must complete before prewarming a paged-KV-stride variant." + ) + if not self.use_tensor_cores: + raise RuntimeError("Paged-KV-stride prewarm requires tensor-core decode.") + if self._jit_module is not None: + raise RuntimeError( + "Paged-KV-stride prewarm is not supported for a custom JIT module." + ) + if self._backend != "fa2": + raise RuntimeError( + "Paged-KV-stride prewarm requires a standard FA2 plan, " + f"got backend={self._backend!r}." + ) + self._cached_module.prewarm_paged_kv_stride_variant(variant) + begin_forward = plan def forward( diff --git a/flashinfer/jit/attention/modules.py b/flashinfer/jit/attention/modules.py index 9571d0cfdf1..bfd73350dc1 100755 --- a/flashinfer/jit/attention/modules.py +++ b/flashinfer/jit/attention/modules.py @@ -15,7 +15,7 @@ """ import os -from typing import Any, List, Optional +from typing import Any, List, Literal, Optional import jinja2 import torch @@ -104,6 +104,62 @@ def plan(self, *args: object) -> object: return self.plan_with_staged_workspace_bytes(*args)[0] +BatchPrefillPagedKVStrideMode = Literal["runtime", "equal", "independent"] +BatchPrefillModuleSurface = Literal["full", "paged"] + +_BATCH_PREFILL_MODULE_URI_SUFFIX = { + ("runtime", "full"): "", + ("equal", "full"): "_kv_stride_equal", + ("independent", "full"): "_kv_stride_independent", + ("independent", "paged"): "_paged_kv_stride_independent", +} + + +def _validate_batch_prefill_module_mode( + backend: str, + paged_kv_stride_mode: BatchPrefillPagedKVStrideMode, + module_surface: BatchPrefillModuleSurface, +) -> None: + if paged_kv_stride_mode not in ("runtime", "equal", "independent"): + raise ValueError( + "paged_kv_stride_mode must be one of 'runtime', 'equal', or " + f"'independent', got {paged_kv_stride_mode!r}" + ) + if module_surface not in ("full", "paged"): + raise ValueError( + f"module_surface must be either 'full' or 'paged', got {module_surface!r}" + ) + if (paged_kv_stride_mode, module_surface) not in _BATCH_PREFILL_MODULE_URI_SUFFIX: + raise ValueError( + "Unsupported batch-prefill mode/surface combination: " + f"mode={paged_kv_stride_mode!r}, surface={module_surface!r}" + ) + if backend != "fa2" and (paged_kv_stride_mode, module_surface) != ( + "runtime", + "full", + ): + raise ValueError( + "Paged KV stride specialization is supported only for the FA2 " + "backend, got " + f"backend={backend!r}, mode={paged_kv_stride_mode!r}, " + f"surface={module_surface!r}" + ) + + +def _get_batch_prefill_module_uri( + base_uri: str, + backend: str, + paged_kv_stride_mode: BatchPrefillPagedKVStrideMode, + module_surface: BatchPrefillModuleSurface, +) -> str: + """Return a content-unique URI for one validated batch-prefill artifact.""" + _validate_batch_prefill_module_mode(backend, paged_kv_stride_mode, module_surface) + return ( + base_uri + + _BATCH_PREFILL_MODULE_URI_SUFFIX[(paged_kv_stride_mode, module_surface)] + ) + + def get_single_decode_uri( dtype_q: torch.dtype, dtype_kv: torch.dtype, @@ -1022,7 +1078,7 @@ def gen_batch_decode_module( ) -def gen_batch_prefill_module( +def _gen_batch_prefill_module( backend: str, dtype_q: torch.dtype, dtype_kv: torch.dtype, @@ -1034,8 +1090,11 @@ def gen_batch_prefill_module( use_sliding_window: bool, use_logits_soft_cap: bool, use_fp16_qk_reduction: bool, + *, + paged_kv_stride_mode: BatchPrefillPagedKVStrideMode, + module_surface: BatchPrefillModuleSurface, ) -> JitSpec: - uri = get_batch_prefill_uri( + base_uri = get_batch_prefill_uri( backend, dtype_q, dtype_kv, @@ -1048,6 +1107,9 @@ def gen_batch_prefill_module( use_logits_soft_cap, use_fp16_qk_reduction, ) + uri = _get_batch_prefill_module_uri( + base_uri, backend, paged_kv_stride_mode, module_surface + ) # use `fp8_enabled` flag to use separate kernel template # this is used for fp8 tensor core computation @@ -1148,6 +1210,132 @@ def gen_batch_prefill_module( use_logits_soft_cap=use_logits_soft_cap, use_fp16_qk_reduction=use_fp16_qk_reduction, fp8_enabled=fp8_enabled, + paged_kv_stride_mode=paged_kv_stride_mode, + module_surface=module_surface, + ) + + +def gen_batch_prefill_module( + backend: str, + dtype_q: torch.dtype, + dtype_kv: torch.dtype, + dtype_o: torch.dtype, + dtype_idx: torch.dtype, + head_dim_qk: int, + head_dim_vo: int, + pos_encoding_mode: int, + use_sliding_window: bool, + use_logits_soft_cap: bool, + use_fp16_qk_reduction: bool, +) -> JitSpec: + """Generate the public full batch-prefill module with runtime stride dispatch.""" + return _gen_batch_prefill_module( + backend, + dtype_q, + dtype_kv, + dtype_o, + dtype_idx, + head_dim_qk, + head_dim_vo, + pos_encoding_mode, + use_sliding_window, + use_logits_soft_cap, + use_fp16_qk_reduction, + paged_kv_stride_mode="runtime", + module_surface="full", + ) + + +def _gen_batch_prefill_primary_module( + backend: str, + dtype_q: torch.dtype, + dtype_kv: torch.dtype, + dtype_o: torch.dtype, + dtype_idx: torch.dtype, + head_dim_qk: int, + head_dim_vo: int, + pos_encoding_mode: int, + use_sliding_window: bool, + use_logits_soft_cap: bool, + use_fp16_qk_reduction: bool, +) -> JitSpec: + """Generate the internal full FA2 primary with equal-stride paged kernels.""" + return _gen_batch_prefill_module( + backend, + dtype_q, + dtype_kv, + dtype_o, + dtype_idx, + head_dim_qk, + head_dim_vo, + pos_encoding_mode, + use_sliding_window, + use_logits_soft_cap, + use_fp16_qk_reduction, + paged_kv_stride_mode="equal", + module_surface="full", + ) + + +def _gen_batch_prefill_independent_full_module( + backend: str, + dtype_q: torch.dtype, + dtype_kv: torch.dtype, + dtype_o: torch.dtype, + dtype_idx: torch.dtype, + head_dim_qk: int, + head_dim_vo: int, + pos_encoding_mode: int, + use_sliding_window: bool, + use_logits_soft_cap: bool, + use_fp16_qk_reduction: bool, +) -> JitSpec: + """Generate the feasibility-only full FA2 independent-stride module.""" + return _gen_batch_prefill_module( + backend, + dtype_q, + dtype_kv, + dtype_o, + dtype_idx, + head_dim_qk, + head_dim_vo, + pos_encoding_mode, + use_sliding_window, + use_logits_soft_cap, + use_fp16_qk_reduction, + paged_kv_stride_mode="independent", + module_surface="full", + ) + + +def _gen_batch_prefill_independent_paged_module( + backend: str, + dtype_q: torch.dtype, + dtype_kv: torch.dtype, + dtype_o: torch.dtype, + dtype_idx: torch.dtype, + head_dim_qk: int, + head_dim_vo: int, + pos_encoding_mode: int, + use_sliding_window: bool, + use_logits_soft_cap: bool, + use_fp16_qk_reduction: bool, +) -> JitSpec: + """Generate the internal paged-only FA2 independent-stride module.""" + return _gen_batch_prefill_module( + backend, + dtype_q, + dtype_kv, + dtype_o, + dtype_idx, + head_dim_qk, + head_dim_vo, + pos_encoding_mode, + use_sliding_window, + use_logits_soft_cap, + use_fp16_qk_reduction, + paged_kv_stride_mode="independent", + module_surface="paged", ) @@ -1196,6 +1384,8 @@ def gen_batch_prefill_attention_sink_module( use_logits_soft_cap=False, use_fp16_qk_reduction=False, fp8_enabled=False, + paged_kv_stride_mode="independent" if backend == "fa2" else "runtime", + module_surface="full", ) @@ -1679,7 +1869,10 @@ def gen_customize_batch_prefill_module( use_logits_soft_cap: bool = False, use_fp16_qk_reduction: bool = False, fp8_enabled: bool = False, + paged_kv_stride_mode: BatchPrefillPagedKVStrideMode = "runtime", + module_surface: BatchPrefillModuleSurface = "full", ) -> JitSpec: + _validate_batch_prefill_module_mode(backend, paged_kv_stride_mode, module_surface) require_fp4_kv_cache = dtype_map_kv[dtype_kv] == "__nv_fp4x2_e2m1" if require_fp4_kv_cache: missing_sf_tensors = [ @@ -1708,6 +1901,12 @@ def gen_customize_batch_prefill_module( "use_sliding_window": str(use_sliding_window).lower(), "use_logits_soft_cap": str(use_logits_soft_cap).lower(), "use_fp16_qk_reduction": str(use_fp16_qk_reduction).lower(), + "paged_kv_stride_mode": paged_kv_stride_mode, + "same_kv_strides_values": { + "runtime": ["true", "false"], + "equal": ["true"], + "independent": ["false"], + }[paged_kv_stride_mode], } if backend == "auto": raise ValueError("backend should not be auto when jit_args is provided") @@ -1735,10 +1934,12 @@ def gen_customize_batch_prefill_module( ) as f: paged_kernel_inst_templ = jinja2.Template(f.read()) - with open( - jit_env.FLASHINFER_CSRC_DIR / "batch_prefill_ragged_kernel_inst.jinja" - ) as f: - ragged_kernel_inst_templ = jinja2.Template(f.read()) + ragged_kernel_inst_templ = None + if module_surface == "full": + with open( + jit_env.FLASHINFER_CSRC_DIR / "batch_prefill_ragged_kernel_inst.jinja" + ) as f: + ragged_kernel_inst_templ = jinja2.Template(f.read()) kwargs |= { "additional_params_decl": additional_params_decl, @@ -1763,20 +1964,26 @@ def gen_customize_batch_prefill_module( ) write_if_different(dest_path, source) - dest_path = ( - gen_directory / f"batch_prefill_ragged_kernel_mask_{mask_mode}.cu" - ) - source_paths.append(dest_path) - source = ragged_kernel_inst_templ.render( - mask_mode=mask_mode_literal[mask_mode], - **kwargs, - ) - write_if_different(dest_path, source) - - for filename in [ - "batch_prefill.cu", - "batch_prefill_jit_binding.cu", - ]: + if ragged_kernel_inst_templ is not None: + dest_path = ( + gen_directory / f"batch_prefill_ragged_kernel_mask_{mask_mode}.cu" + ) + source_paths.append(dest_path) + source = ragged_kernel_inst_templ.render( + mask_mode=mask_mode_literal[mask_mode], + **kwargs, + ) + write_if_different(dest_path, source) + + host_sources = ( + ["batch_prefill.cu", "batch_prefill_jit_binding.cu"] + if module_surface == "full" + else [ + "batch_prefill_paged.cu", + "batch_prefill_paged_jit_binding.cu", + ] + ) + for filename in host_sources: src_path = jit_env.FLASHINFER_CSRC_DIR / filename dest_path = gen_directory / filename source_paths.append(dest_path) @@ -1784,6 +1991,11 @@ def gen_customize_batch_prefill_module( source = f.read() write_if_different(dest_path, source) + paged_header = "batch_prefill_paged.cuh" + with open(jit_env.FLASHINFER_CSRC_DIR / paged_header, "r") as f: + source = f.read() + write_if_different(gen_directory / paged_header, source) + generated_config_path = gen_directory / "batch_prefill_config.inc" write_if_different(generated_config_path, generated_inc_str) extra_cuda_cflags = _fa2_prefill_head_dim_nvcc_flags( diff --git a/flashinfer/prefill.py b/flashinfer/prefill.py index ccf192e9ca8..ccfb9e019e8 100755 --- a/flashinfer/prefill.py +++ b/flashinfer/prefill.py @@ -17,6 +17,7 @@ import functools import logging import math +import threading from types import SimpleNamespace from typing import Any, Dict, List, Literal, Optional, Tuple, Union, overload @@ -25,6 +26,7 @@ from .api_logging import flashinfer_api from .cudnn import cudnn_batch_prefill_with_kv_cache from .jit import ( + MissingJITCacheError, gen_batch_prefill_module, gen_customize_batch_prefill_module, gen_fmha_cutlass_sm100a_module, @@ -36,6 +38,12 @@ get_single_prefill_uri, setup_cubin_loader, ) +from .jit.attention.modules import ( + BatchPrefillModuleSurface, + BatchPrefillPagedKVStrideMode, + _gen_batch_prefill_independent_paged_module, + _gen_batch_prefill_primary_module, +) from .jit.attention.utils import _is_nvfp4_kv_dtype from .page import get_seq_lens from .quantization import packbits, segment_packbits @@ -196,6 +204,8 @@ def get_customize_batch_prefill_module( use_logits_soft_cap: bool = False, use_fp16_qk_reduction: bool = False, fp8_enabled: bool = False, + paged_kv_stride_mode: BatchPrefillPagedKVStrideMode = "runtime", + module_surface: BatchPrefillModuleSurface = "full", ): return gen_customize_batch_prefill_module( backend, @@ -217,6 +227,8 @@ def get_customize_batch_prefill_module( use_logits_soft_cap, use_fp16_qk_reduction, fp8_enabled, + paged_kv_stride_mode=paged_kv_stride_mode, + module_surface=module_surface, ).build_and_load() @@ -455,8 +467,65 @@ def _fake_run_single_prefill( return SimpleNamespace(run=run_single_prefill) +class _LazyBatchPrefillIndependentModule: + """Load one independent-stride batch-prefill module on first eager use.""" + + def __init__(self, spec: Any) -> None: + self._spec = spec + self._lock = threading.Lock() + self._module: Optional[Any] = None + + @property + def is_loaded(self) -> bool: + """Whether this holder already has an in-process loaded module.""" + return self._module is not None + + @staticmethod + def _check_not_capturing() -> None: + if torch.cuda.is_current_stream_capturing(): + raise RuntimeError( + "The lazy independent paged-KV-stride module cannot be compiled " + "or loaded during CUDA graph capture. Call " + "prewarm_paged_kv_stride_variant('independent') after plan() " + "and before capture." + ) + + def get(self) -> Any: + """Return the loaded module, building it once outside graph capture.""" + module = self._module + if module is not None: + return module + + self._check_not_capturing() + with self._lock: + module = self._module + if module is not None: + return module + self._check_not_capturing() + try: + module = self._spec.build_and_load() + except MissingJITCacheError as exc: + raise MissingJITCacheError( + "Unequal K/V data strides require FlashInfer's lazy " + "independent paged module, which is not included in the " + "default JIT cache. Use equal-stride K/V tensors, enable " + "local JIT and call " + "prewarm_paged_kv_stride_variant('independent') after " + "plan(), or install a compatible independent-module cache " + "package when one becomes available.", + spec=exc.spec, + ) from exc + self._module = module + return module + + def prewarm(self) -> None: + """Eagerly load the independent module for later graph capture.""" + self.get() + + @functools.cache def get_batch_prefill_module(backend, *args): + lazy_independent_module: Optional[_LazyBatchPrefillIndependentModule] = None if backend == "trtllm-gen": uri = "trtllm_gen_context" module = get_trtllm_gen_prefill_module() @@ -464,6 +533,16 @@ def get_batch_prefill_module(backend, *args): workspace_size_func = None ragged_run_func = module.ragged_run paged_run_func = module.paged_run + elif backend == "fa2": + uri = get_batch_prefill_uri(backend, *args) + module = _gen_batch_prefill_primary_module(backend, *args).build_and_load() + lazy_independent_module = _LazyBatchPrefillIndependentModule( + _gen_batch_prefill_independent_paged_module(backend, *args) + ) + plan_func = module.plan + workspace_size_func = getattr(module, "workspace_size", None) + ragged_run_func = module.ragged_run + paged_run_func = module.paged_run else: uri = get_batch_prefill_uri(backend, *args) module = gen_batch_prefill_module(backend, *args).build_and_load() @@ -748,7 +827,12 @@ def paged_run( ) elif backend == "fa2": assert not is_float8(q) - paged_run_func( + if tuple(paged_k_cache.stride()) == tuple(paged_v_cache.stride()): + routed_paged_run_func = paged_run_func + else: + assert lazy_independent_module is not None + routed_paged_run_func = lazy_independent_module.get().paged_run + routed_paged_run_func( float_workspace_buffer, int_workspace_buffer, plan_info_vec, @@ -896,11 +980,23 @@ def _fake_paged_run( # # Note that plan is not part of model logic. It should not be included in # Cuda Graph or torch.compile. So, we don't provide a torch library for plan. + def prewarm_paged_kv_stride_variant(variant: str = "independent") -> None: + """Load the requested lazy paged-KV-stride variant before graph capture.""" + if variant != "independent": + raise ValueError(f"variant must be 'independent', got {variant!r}") + if lazy_independent_module is None: + raise RuntimeError( + "Paged-KV-stride prewarm is available only for standard FA2 " + f"batch-prefill modules, got backend={backend!r}." + ) + lazy_independent_module.prewarm() + return SimpleNamespace( plan=plan_func, workspace_size=workspace_size_func, ragged_run=ragged_run, paged_run=paged_run, + prewarm_paged_kv_stride_variant=prewarm_paged_kv_stride_variant, ) @@ -2754,6 +2850,38 @@ def plan( self._seq_lens_kv = seq_lens self._seq_lens_q = seq_lens_q if seq_lens_q is not None else seq_lens + @flashinfer_api + def prewarm_paged_kv_stride_variant(self, variant: str = "independent") -> None: + r"""Load a lazy paged-KV-stride variant after :meth:`plan`. + + Call this method before CUDA graph capture when a planned standard FA2 + wrapper will receive K and V tensors with different data strides. + + Parameters + ---------- + variant : str + The paged-KV-stride variant to prewarm. The only supported value is + ``"independent"`` (the default), for K and V with different data strides. + """ + if ( + getattr(self, "_plan_info", None) is None + or getattr(self, "_cached_module", None) is None + ): + raise RuntimeError( + "plan() must complete before prewarming a paged-KV-stride variant." + ) + if self._jit_module is not None: + raise RuntimeError( + "Paged-KV-stride prewarm is not supported for a custom JIT module." + ) + if self._backend != "fa2": + raise RuntimeError( + "Paged-KV-stride prewarm requires a standard FA2 plan, " + f"got backend={self._backend!r}." + ) + assert self._cached_module is not None + self._cached_module.prewarm_paged_kv_stride_variant(variant) + begin_forward = plan def forward( diff --git a/include/flashinfer/attention/prefill.cuh b/include/flashinfer/attention/prefill.cuh index 34e6cd62a60..8d17c467472 100644 --- a/include/flashinfer/attention/prefill.cuh +++ b/include/flashinfer/attention/prefill.cuh @@ -3796,7 +3796,7 @@ __device__ __forceinline__ void vosplit_write_o( } } -template +template __device__ __forceinline__ void BatchPrefillWithPagedKVCacheDevice( const Params params, SmemStorage& smem_storage, const dim3 tid = threadIdx, const uint32_t bx = blockIdx.x, const uint32_t kv_head_idx = blockIdx.z, @@ -4010,8 +4010,9 @@ __device__ __forceinline__ void BatchPrefillWithPagedKVCacheDevice( // smem path computes offsets on the fly, so the arrays collapse to [1] stubs there. [[maybe_unused]] size_t thr_local_kv_offset_k[KTraits::USE_KV_SHARED_SMEM ? 1 : NUM_PAGED_KV_OFFSETS]; - [[maybe_unused]] size_t - thr_local_kv_offset_v[KTraits::USE_KV_SHARED_SMEM ? 1 : NUM_PAGED_KV_OFFSETS]; + [[maybe_unused]] size_t thr_local_kv_offset_v[KTraits::USE_KV_SHARED_SMEM || SAME_KV_STRIDES + ? 1 + : NUM_PAGED_KV_OFFSETS]; uint32_t k_smem_offset_r = k_smem.template get_permuted_offset( get_warp_idx_kv(tid.z) * NUM_MMA_KV * 16 + 8 * (lane_idx / 16) + @@ -4070,8 +4071,10 @@ __device__ __forceinline__ void BatchPrefillWithPagedKVCacheDevice( (lane_idx % KV_THR_LAYOUT_COL) * upcast_size() / fp4_pack_factor; thr_local_kv_offset_k[i] = paged_kv.protective_get_k_offset( page_iter, kv_head_idx, entry_idx, feat_idx, last_indptr); - thr_local_kv_offset_v[i] = paged_kv.protective_get_v_offset( - page_iter, kv_head_idx, entry_idx, feat_idx, last_indptr); + if constexpr (!SAME_KV_STRIDES) { + thr_local_kv_offset_v[i] = paged_kv.protective_get_v_offset( + page_iter, kv_head_idx, entry_idx, feat_idx, last_indptr); + } } page_produce_kv(&smem_storage, &k_smem_offset_w, paged_kv.k_data, 0, thr_local_kv_offset_k, chunk_size, warp_idx, lane_idx); @@ -4083,8 +4086,13 @@ __device__ __forceinline__ void BatchPrefillWithPagedKVCacheDevice( cp_async::commit_group(); // Shared K/V loads V(0) inside iter 0 after Q.K^T; preloading it would clobber K(0). if constexpr (!KTraits::USE_KV_SHARED_SMEM) { - page_produce_kv(&smem_storage, &v_smem_offset_w, paged_kv.v_data, 0, - thr_local_kv_offset_v, chunk_size, warp_idx, lane_idx); + if constexpr (SAME_KV_STRIDES) { + page_produce_kv(&smem_storage, &v_smem_offset_w, paged_kv.v_data, 0, + thr_local_kv_offset_k, chunk_size, warp_idx, lane_idx); + } else { + page_produce_kv(&smem_storage, &v_smem_offset_w, paged_kv.v_data, 0, + thr_local_kv_offset_v, chunk_size, warp_idx, lane_idx); + } page_produce_kv_sf(&smem_storage, maybe_v_cache_sf, packed_page_iter_base, last_indptr * (uint32_t)paged_kv.page_size, kv_head_idx, v_sf_stride_page, v_sf_stride_h, v_sf_stride_n, @@ -4177,8 +4185,10 @@ __device__ __forceinline__ void BatchPrefillWithPagedKVCacheDevice( (lane_idx % KV_THR_LAYOUT_COL) * upcast_size() / fp4_pack_factor; thr_local_kv_offset_k[i] = paged_kv.protective_get_k_offset( page_iter, kv_head_idx, entry_idx, feat_idx, last_indptr); - thr_local_kv_offset_v[i] = paged_kv.protective_get_v_offset( - page_iter, kv_head_idx, entry_idx, feat_idx, last_indptr); + if constexpr (!SAME_KV_STRIDES) { + thr_local_kv_offset_v[i] = paged_kv.protective_get_v_offset( + page_iter, kv_head_idx, entry_idx, feat_idx, last_indptr); + } } } // Shared K/V serializes loads (no K/V prefetch overlap) -> drain fully. @@ -4331,9 +4341,15 @@ __device__ __forceinline__ void BatchPrefillWithPagedKVCacheDevice( cp_async::commit_group(); packed_page_iter_base = next_packed_page_iter_base; } else { - page_produce_kv(&smem_storage, &v_smem_offset_w, paged_kv.v_data, - (iter + 1) * CTA_TILE_KV, thr_local_kv_offset_v, - chunk_size, warp_idx, lane_idx); + if constexpr (SAME_KV_STRIDES) { + page_produce_kv(&smem_storage, &v_smem_offset_w, paged_kv.v_data, + (iter + 1) * CTA_TILE_KV, thr_local_kv_offset_k, + chunk_size, warp_idx, lane_idx); + } else { + page_produce_kv(&smem_storage, &v_smem_offset_w, paged_kv.v_data, + (iter + 1) * CTA_TILE_KV, thr_local_kv_offset_v, + chunk_size, warp_idx, lane_idx); + } page_produce_kv_sf( &smem_storage, maybe_v_cache_sf, packed_page_iter_base, last_indptr * (uint32_t)paged_kv.page_size, kv_head_idx, v_sf_stride_page, @@ -4412,12 +4428,12 @@ __device__ __forceinline__ void BatchPrefillWithPagedKVCacheDevice( #endif } -template +template __global__ __launch_bounds__(KTraits::NUM_THREADS) void BatchPrefillWithPagedKVCacheKernel( const __grid_constant__ Params params) { extern __shared__ uint8_t smem[]; auto& smem_storage = reinterpret_cast(smem); - BatchPrefillWithPagedKVCacheDevice(params, smem_storage); + BatchPrefillWithPagedKVCacheDevice(params, smem_storage); } template cudaError_t BatchPrefillWithPagedKVCacheDispatchedImpl(Params params, @@ -4767,7 +4783,7 @@ cudaError_t BatchPrefillWithPagedKVCacheDispatchedImpl(Params params, FLASHINFER_ERROR(err_msg.str()); } else { size_t smem_size = sizeof(typename KTraits::SharedStoragePaged); - auto kernel = BatchPrefillWithPagedKVCacheKernel; + auto kernel = BatchPrefillWithPagedKVCacheKernel; // Exact final check: the analytic NUM_MMA_KV budget can slightly // under-count the real struct (cross-warp merge buffers, padding); // fail with a clear error instead of a launch-time cudaErrorInvalidValue. @@ -4834,17 +4850,17 @@ cudaError_t BatchPrefillWithPagedKVCacheDispatchedImpl(Params params, return cudaSuccess; } -template cudaError_t BatchPrefillWithPagedKVCacheDispatched(Params params, typename Params::DTypeO* tmp_v, float* tmp_s, bool enable_pdl, cudaStream_t stream) { -#define FLASHINFER_PAGED_PREFILL_CALL(EN) \ - (BatchPrefillWithPagedKVCacheDispatchedImpl(params, tmp_v, tmp_s, \ - enable_pdl, stream)) +#define FLASHINFER_PAGED_PREFILL_CALL(EN) \ + (BatchPrefillWithPagedKVCacheDispatchedImpl< \ + SAME_KV_STRIDES, CTA_TILE_Q, HEAD_DIM_QK, HEAD_DIM_VO, POS_ENCODING_MODE, \ + USE_FP16_QK_REDUCTION, MASK_MODE, AttentionVariant, Params, EN>(params, tmp_v, tmp_s, \ + enable_pdl, stream)) FLASHINFER_DISPATCH_FP4_REPACK(typename Params::DTypeKV, CTA_TILE_Q, HEAD_DIM_QK, HEAD_DIM_VO, FLASHINFER_PAGED_PREFILL_CALL) #undef FLASHINFER_PAGED_PREFILL_CALL diff --git a/tests/attention/test_batch_prefill_kernels.py b/tests/attention/test_batch_prefill_kernels.py index 5804fcb3328..b1cff9a3705 100644 --- a/tests/attention/test_batch_prefill_kernels.py +++ b/tests/attention/test_batch_prefill_kernels.py @@ -20,6 +20,7 @@ import pytest import torch from tests.test_helpers.jit_utils import gen_prefill_attention_modules +from tests.test_helpers.paged_kv import make_padded_paged_kv_view import flashinfer from tests.test_helpers.test_helpers import assert_close_chunked, ref_single_prefill @@ -338,6 +339,173 @@ def test_batch_prefill_with_paged_kv_cache( _assert_no_ref_mismatch(mismatch_counts) +@pytest.mark.parametrize("kv_layout,head_dim", [("NHD", 64), ("HND", 128)]) +def test_batch_prefill_lazy_stride_router_plan_reuse(kv_layout, head_dim): + """Reuse one equal-primary plan across equal/unequal/equal paged runs.""" + torch.manual_seed(42) + batch_size, qo_len, kv_len, page_size = 2, 17, 97, 16 + num_qo_heads, num_kv_heads = 8, 2 + pages_per_request = (kv_len + page_size - 1) // page_size + total_pages = batch_size * pages_per_request + + q = torch.randn( + batch_size * qo_len, + num_qo_heads, + head_dim, + device="cuda", + dtype=torch.bfloat16, + ) + if kv_layout == "NHD": + cache_shape = (total_pages, page_size, num_kv_heads, head_dim) + to_dense = lambda cache: cache.reshape(-1, num_kv_heads, head_dim) + else: + cache_shape = (total_pages, num_kv_heads, page_size, head_dim) + to_dense = lambda cache: cache.permute(0, 2, 1, 3).reshape( + -1, num_kv_heads, head_dim + ) + + k = torch.randn(cache_shape, device="cuda", dtype=torch.bfloat16) / 4 + v_equal = torch.randn(cache_shape, device="cuda", dtype=torch.bfloat16) / 4 + v_unequal = make_padded_paged_kv_view(v_equal, kv_layout) + assert k.shape == v_equal.shape == v_unequal.shape + assert k.stride() == v_equal.stride() + assert k.stride() != v_unequal.stride() + + qo_indptr = ( + torch.arange(0, batch_size + 1, device="cuda", dtype=torch.int32) * qo_len + ) + kv_indptr = ( + torch.arange(0, batch_size + 1, device="cuda", dtype=torch.int32) + * pages_per_request + ) + kv_indices = torch.arange(total_pages, device="cuda", dtype=torch.int32) + last_page_len = torch.full( + (batch_size,), + (kv_len - 1) % page_size + 1, + device="cuda", + dtype=torch.int32, + ) + + wrapper = flashinfer.BatchPrefillWithPagedKVCacheWrapper( + torch.empty(128 * 1024 * 1024, device="cuda", dtype=torch.uint8), + kv_layout, + backend="fa2", + ) + wrapper.plan( + qo_indptr, + kv_indptr, + kv_indices, + last_page_len, + num_qo_heads, + num_kv_heads, + head_dim, + page_size, + causal=True, + q_data_type=torch.bfloat16, + kv_data_type=torch.bfloat16, + fixed_split_size=2, + ) + plan_info = tuple(wrapper._plan_info) + outputs = [ + wrapper.run(q, cache) for cache in ((k, v_equal), (k, v_unequal), (k, v_equal)) + ] + assert tuple(wrapper._plan_info) == plan_info + + expected_batches = [] + for batch_idx in range(batch_size): + q_i = q[batch_idx * qo_len : (batch_idx + 1) * qo_len] + page_slice = slice( + batch_idx * pages_per_request, + (batch_idx + 1) * pages_per_request, + ) + expected_i, _ = ref_single_prefill( + q_i, + to_dense(k[page_slice])[:kv_len], + to_dense(v_equal[page_slice])[:kv_len], + causal=True, + ) + expected_batches.append(expected_i) + expected = torch.cat(expected_batches) + for output in outputs: + torch.testing.assert_close(output, expected, rtol=2e-2, atol=2e-2) + torch.testing.assert_close(outputs[0], outputs[1], rtol=1e-2, atol=1e-2) + torch.testing.assert_close(outputs[0], outputs[2], rtol=1e-2, atol=1e-2) + + +def test_batch_prefill_lazy_stride_router_nvfp4(): + """Route on data strides while preserving independent NVFP4 scale strides.""" + torch.manual_seed(42) + batch_size, qo_len, kv_len, page_size = 1, 17, 33, 16 + num_qo_heads, num_kv_heads, head_dim = 4, 2, 128 + pages_per_request = (kv_len + page_size - 1) // page_size + total_pages = batch_size * pages_per_request + + q = torch.randn( + batch_size * qo_len, + num_qo_heads, + head_dim, + device="cuda", + dtype=torch.float16, + ) + packed_shape = (total_pages, page_size, num_kv_heads, head_dim // 2) + k_packed, k_sf, k_scale = create_nvfp4_kv(packed_shape, "cuda") + v_packed, v_sf, v_scale = create_nvfp4_kv(packed_shape, "cuda") + v_packed_unequal = make_padded_paged_kv_view(v_packed, "NHD") + k_sf_unequal = make_padded_paged_kv_view(k_sf, "NHD", padding_heads=1) + v_sf_unequal = make_padded_paged_kv_view(v_sf, "NHD", padding_heads=3) + assert k_packed.stride() == v_packed.stride() + assert k_packed.stride() != v_packed_unequal.stride() + assert k_sf_unequal.stride() != v_sf_unequal.stride() + + qo_indptr = torch.tensor([0, qo_len], device="cuda", dtype=torch.int32) + kv_indptr = torch.tensor([0, pages_per_request], device="cuda", dtype=torch.int32) + kv_indices = torch.arange(total_pages, device="cuda", dtype=torch.int32) + last_page_len = torch.tensor( + [(kv_len - 1) % page_size + 1], device="cuda", dtype=torch.int32 + ) + custom_mask = torch.tril( + torch.ones(qo_len, kv_len, device="cuda", dtype=torch.bool), + diagonal=kv_len - qo_len, + ).flatten() + wrapper = flashinfer.BatchPrefillWithPagedKVCacheWrapper( + torch.empty(128 * 1024 * 1024, device="cuda", dtype=torch.uint8), + "NHD", + backend="fa2", + ) + wrapper.plan( + qo_indptr, + kv_indptr, + kv_indices, + last_page_len, + num_qo_heads, + num_kv_heads, + head_dim, + page_size, + custom_mask=custom_mask, + q_data_type=torch.float16, + kv_data_type=torch.uint8, + ) + plan_info = tuple(wrapper._plan_info) + equal_output = wrapper.run( + q, + (k_packed, v_packed), + kv_cache_sf=(k_sf_unequal, v_sf_unequal), + k_scale=k_scale.item(), + v_scale=v_scale.item(), + ) + + wrapper.prewarm_paged_kv_stride_variant("independent") + unequal_output = wrapper.run( + q, + (k_packed, v_packed_unequal), + kv_cache_sf=(k_sf_unequal, v_sf_unequal), + k_scale=k_scale.item(), + v_scale=v_scale.item(), + ) + assert tuple(wrapper._plan_info) == plan_info + torch.testing.assert_close(unequal_output, equal_output, rtol=1e-3, atol=1e-3) + + @pytest.mark.parametrize("causal", [False, True]) @pytest.mark.parametrize("pos_encoding_mode", ["NONE", "ROPE_LLAMA"]) def test_batch_prefill_with_paged_kv_cache_head_dim_512( diff --git a/tests/jit/test_attention_utils.py b/tests/jit/test_attention_utils.py index 4ddc0672efb..cc5825c0d58 100644 --- a/tests/jit/test_attention_utils.py +++ b/tests/jit/test_attention_utils.py @@ -1,15 +1,84 @@ from pathlib import Path +import pytest import torch +from flashinfer import prefill +from flashinfer.jit import core from flashinfer.jit import env as jit_env +from flashinfer.jit.attention import modules as attention_modules from flashinfer.jit.attention.modules import gen_customize_batch_prefill_module +@pytest.fixture +def generated_sources(tmp_path, monkeypatch): + monkeypatch.setattr(core, "check_cuda_arch", lambda: None) + monkeypatch.setattr( + attention_modules.current_compilation_context, "TARGET_CUDA_ARCHS", {(9, 0)} + ) + monkeypatch.setattr(jit_env, "FLASHINFER_GEN_SRC_DIR", tmp_path) + monkeypatch.setattr( + jit_env, "FLASHINFER_CSRC_DIR", Path(__file__).resolve().parents[2] / "csrc" + ) + return tmp_path + + +@pytest.mark.parametrize( + "backend,mode", + [("fa2", None), ("fa2", "independent"), ("fa3", None), ("fa3", "runtime")], +) +def test_custom_prefill_adapter_preserves_stride_specialization( + generated_sources, monkeypatch, backend, mode +): + # Exercise the cached adapter and real generator; only compilation is stubbed. + monkeypatch.setattr(core.JitSpec, "build_and_load", lambda self: self) + uri = f"test_custom_prefill_adapter_{backend}_{mode}_{generated_sources.name}" + kwargs = ( + {} if mode is None else {"paged_kv_stride_mode": mode, "module_surface": "full"} + ) + spec = prefill.get_customize_batch_prefill_module( + backend, + uri, + torch.bfloat16, + torch.bfloat16, + torch.bfloat16, + torch.int32, + 128, + 128, + [], + [], + [], + [], + "DefaultAttention", + "struct DefaultAttention {};", + **kwargs, + ) + assert spec.name == uri + assert len(spec.sources) == 10 + generated_dir = generated_sources / uri + suffix = "_sm90" if backend == "fa3" else "" + binding = (generated_dir / f"batch_prefill{suffix}_jit_binding.cu").read_text() + assert "TVM_FFI_DLL_EXPORT_TYPED_FUNC(paged_run," in binding + assert "TVM_FFI_DLL_EXPORT_TYPED_FUNC(ragged_run," in binding + if backend == "fa2": + expected_mode = "INDEPENDENT" if mode == "independent" else "RUNTIME" + config = (generated_dir / "batch_prefill_config.inc").read_text() + assert ( + f"#define PAGED_KV_STRIDE_MODE PAGED_KV_STRIDE_MODE_{expected_mode}" + in config + ) + + def test_batch_prefill_nvfp4_swa_paged_params_declares_sf_strides( tmp_path, monkeypatch ): repo_root = Path(__file__).resolve().parents[2] + monkeypatch.setattr(core, "check_cuda_arch", lambda: None) + monkeypatch.setattr( + attention_modules.current_compilation_context, + "TARGET_CUDA_ARCHS", + {(10, 3)}, + ) monkeypatch.setattr(jit_env, "FLASHINFER_GEN_SRC_DIR", tmp_path / "generated") monkeypatch.setattr(jit_env, "FLASHINFER_CSRC_DIR", repo_root / "csrc") @@ -55,6 +124,120 @@ def test_batch_prefill_nvfp4_swa_paged_params_declares_sf_strides( assert f"uint32_t {field}_stride_n;" in generated +@pytest.mark.parametrize( + "factory,suffix,source_count,variants", + [ + ("gen_batch_prefill_module", "", 10, {"true", "false"}), + ("_gen_batch_prefill_primary_module", "_kv_stride_equal", 10, {"true"}), + ( + "_gen_batch_prefill_independent_paged_module", + "_paged_kv_stride_independent", + 6, + {"false"}, + ), + ], +) +def test_batch_prefill_stride_module_surface( + generated_sources, factory, suffix, source_count, variants +): + args = ( + "fa2", + torch.bfloat16, + torch.bfloat16, + torch.bfloat16, + torch.int32, + 64, + 64, + 0, + False, + False, + False, + ) + spec = getattr(attention_modules, factory)(*args) + assert spec.name == attention_modules.get_batch_prefill_uri(*args) + suffix + assert len(spec.sources) == source_count + for mask in range(4): + source = ( + generated_sources / spec.name / f"batch_prefill_paged_kernel_mask_{mask}.cu" + ).read_text() + for variant in ("true", "false"): + assert (f"/*SAME_KV_STRIDES=*/{variant}" in source) == (variant in variants) + binding_name = ( + "batch_prefill_paged_jit_binding.cu" + if source_count == 6 + else "batch_prefill_jit_binding.cu" + ) + binding = (generated_sources / spec.name / binding_name).read_text() + assert "TVM_FFI_DLL_EXPORT_TYPED_FUNC(paged_run," in binding + for entrypoint in ("plan", "workspace_size", "ragged_run"): + assert (f"TVM_FFI_DLL_EXPORT_TYPED_FUNC({entrypoint}," in binding) == ( + source_count == 10 + ) + + +def test_gen_attention_aot_matrix_contains_equal_primaries_only(generated_sources): + import flashinfer.aot as flashinfer_aot + + specs = list( + flashinfer_aot.gen_attention( + f16_dtype_=[torch.bfloat16], + f8_dtype_=[], + fa2_head_dim_=[(64, 64), (128, 128)], + fa3_head_dim_=[], + use_sliding_window_=[False, True], + use_logits_soft_cap_=[False], + has_sm90=False, + has_sm100=False, + add_gemma=False, + add_oai_oss=False, + ) + ) + standard_prefill_specs = [ + spec for spec in specs if spec.name.startswith("batch_prefill_with_kv_cache_") + ] + + assert len(standard_prefill_specs) == 4 + assert all( + spec.name.endswith("_kv_stride_equal") for spec in standard_prefill_specs + ) + assert all(len(spec.sources) == 10 for spec in standard_prefill_specs) + assert not any( + "_paged_kv_stride_independent" in spec.name + or spec.name.endswith("_kv_stride_independent") + for spec in specs + ) + + +def test_attention_sink_stays_independent_full(generated_sources): + args = ( + "fa2", + torch.bfloat16, + torch.bfloat16, + torch.bfloat16, + torch.int32, + 64, + 64, + 0, + False, + ) + spec = attention_modules.gen_batch_prefill_attention_sink_module(*args) + assert spec.name == attention_modules.get_batch_prefill_attention_sink_uri(*args) + assert len(spec.sources) == 10 + + generated_dir = generated_sources / spec.name + config = (generated_dir / "batch_prefill_config.inc").read_text() + assert "#define PAGED_KV_STRIDE_MODE PAGED_KV_STRIDE_MODE_INDEPENDENT" in config + binding = (generated_dir / "batch_prefill_jit_binding.cu").read_text() + for entrypoint in ("plan", "workspace_size", "ragged_run", "paged_run"): + assert f"TVM_FFI_DLL_EXPORT_TYPED_FUNC({entrypoint}," in binding + for mask_mode in range(4): + paged_text = ( + generated_dir / f"batch_prefill_paged_kernel_mask_{mask_mode}.cu" + ).read_text() + assert "/*SAME_KV_STRIDES=*/true" not in paged_text + assert "/*SAME_KV_STRIDES=*/false" in paged_text + + def test_batch_prefill_nvfp4_requires_sf_tensors(): try: gen_customize_batch_prefill_module( diff --git a/tests/jit/test_batch_prefill_lazy_loader.py b/tests/jit/test_batch_prefill_lazy_loader.py new file mode 100644 index 00000000000..cedbc422d2d --- /dev/null +++ b/tests/jit/test_batch_prefill_lazy_loader.py @@ -0,0 +1,283 @@ +from concurrent.futures import ThreadPoolExecutor +from types import SimpleNamespace +import threading +import time + +import pytest +import torch + +import flashinfer.prefill as prefill +from flashinfer.cascade import ( + BatchPrefillWithSharedPrefixPagedKVCacheWrapper, + MultiLevelCascadeAttentionWrapper, +) +from flashinfer.decode import BatchDecodeWithPagedKVCacheWrapper +from flashinfer.jit import MissingJITCacheError + + +_MODULE_ARGS = ( + torch.bfloat16, + torch.bfloat16, + torch.bfloat16, + torch.int32, + 128, + 128, + 0, + False, + False, + False, +) + + +class _FakeSpec: + def __init__(self, module, *, error=None, delay_seconds=0.0): + self.module = module + self.error = error + self.delay_seconds = delay_seconds + self.build_count = 0 + self._counter_lock = threading.Lock() + + def build_and_load(self): + with self._counter_lock: + self.build_count += 1 + if self.delay_seconds: + time.sleep(self.delay_seconds) + if self.error is not None: + raise self.error + return self.module + + +def _raw_module(label, calls): + def paged_run(*args): + calls.append(label) + + return SimpleNamespace( + plan=lambda *args: [1], + workspace_size=lambda *args: 0, + ragged_run=lambda *args: None, + paged_run=paged_run, + ) + + +def _run_paged(module, paged_k_cache, paged_v_cache, **kwargs): + tensor = torch.empty(1) + return module.paged_run( + tensor, + tensor, + [1], + tensor, + paged_k_cache, + paged_v_cache, + tensor, + tensor, + tensor, + tensor, + tensor, + None, + 0, + 0, + -1, + False, + None, + None, + None, + None, + None, + None, + 0.0, + 1.0, + None, + None, + None, + 1.0, + 1.0, + 0, + 0, + **kwargs, + ) + + +@pytest.fixture(autouse=True) +def _clear_batch_prefill_module_cache(): + prefill.get_batch_prefill_module.cache_clear() + yield + prefill.get_batch_prefill_module.cache_clear() + + +def test_lazy_holder_capture_order_warm_fast_path_and_no_jit(monkeypatch): + loaded_module = object() + spec = _FakeSpec(loaded_module) + holder = prefill._LazyBatchPrefillIndependentModule(spec) + capture_states = iter((False, True)) + monkeypatch.setattr( + torch.cuda, "is_current_stream_capturing", lambda: next(capture_states) + ) + + with pytest.raises(RuntimeError, match="before capture"): + holder.get() + assert spec.build_count == 0 + assert not holder.is_loaded + + monkeypatch.setattr(torch.cuda, "is_current_stream_capturing", lambda: False) + holder.prewarm() + assert holder.is_loaded + assert spec.build_count == 1 + + def unexpected_capture_check(): + raise AssertionError("warm fast path must not query capture state") + + monkeypatch.setattr( + torch.cuda, "is_current_stream_capturing", unexpected_capture_check + ) + assert holder.get() is loaded_module + assert spec.build_count == 1 + + missing_spec = object() + no_jit_spec = _FakeSpec( + object(), + error=MissingJITCacheError("generic cache miss", spec=missing_spec), + ) + no_jit_holder = prefill._LazyBatchPrefillIndependentModule(no_jit_spec) + monkeypatch.setattr(torch.cuda, "is_current_stream_capturing", lambda: False) + with pytest.raises(MissingJITCacheError) as exc_info: + no_jit_holder.get() + assert exc_info.value.spec is missing_spec + assert isinstance(exc_info.value.__cause__, MissingJITCacheError) + message = str(exc_info.value) + assert "Unequal K/V data strides" in message + assert "not included in the default JIT cache" in message + assert "prewarm_paged_kv_stride_variant('independent')" in message + + +def test_concurrent_first_use_loads_once(monkeypatch): + loaded_module = object() + spec = _FakeSpec(loaded_module, delay_seconds=0.05) + holder = prefill._LazyBatchPrefillIndependentModule(spec) + monkeypatch.setattr(torch.cuda, "is_current_stream_capturing", lambda: False) + + with ThreadPoolExecutor(max_workers=8) as pool: + modules = list(pool.map(lambda _: holder.get(), range(8))) + + assert all(module is loaded_module for module in modules) + assert spec.build_count == 1 + + +def test_fa2_selects_stride_variant_once(monkeypatch): + calls = [] + primary_spec = _FakeSpec(_raw_module("primary", calls)) + independent_spec = _FakeSpec(_raw_module("independent", calls)) + monkeypatch.setattr( + prefill, "_gen_batch_prefill_primary_module", lambda *args: primary_spec + ) + monkeypatch.setattr( + prefill, + "_gen_batch_prefill_independent_paged_module", + lambda *args: independent_spec, + ) + monkeypatch.setattr(torch.cuda, "is_current_stream_capturing", lambda: False) + + module = prefill.get_batch_prefill_module("fa2", *_MODULE_ARGS) + assert primary_spec.build_count == 1 + assert independent_spec.build_count == 0 + + shape = (2, 3, 4, 5) + equal_k = torch.empty_strided(shape, (60, 20, 5, 1)) + equal_v = torch.empty_strided(shape, (60, 20, 5, 1)) + unequal_v = torch.empty_strided(shape, (80, 20, 5, 1)) + # Scale-factor layouts must not trigger the independent data-stride module. + scales = dict( + key_block_scales=torch.empty_strided(shape, (60, 20, 5, 1), dtype=torch.uint8), + value_block_scales=torch.empty_strided( + shape, (80, 20, 5, 1), dtype=torch.uint8 + ), + ) + _run_paged(module, equal_k, equal_v, **scales) + assert calls == ["primary"] + assert independent_spec.build_count == 0 + _run_paged(module, equal_k, unequal_v) + _run_paged(module, equal_k, equal_v) + + assert calls == ["primary", "independent", "primary"] + assert independent_spec.build_count == 1 + module.prewarm_paged_kv_stride_variant("independent") + assert independent_spec.build_count == 1 + with pytest.raises(ValueError, match="variant must be 'independent'"): + module.prewarm_paged_kv_stride_variant("unknown") + + +def test_non_fa2_keeps_public_full_module_and_rejects_prewarm(monkeypatch): + calls = [] + public_spec = _FakeSpec(_raw_module("public", calls)) + monkeypatch.setattr(prefill, "gen_batch_prefill_module", lambda *args: public_spec) + + def unexpected_specialized_generator(*args): + raise AssertionError("non-FA2 must not construct a stride-specialized module") + + monkeypatch.setattr( + prefill, "_gen_batch_prefill_primary_module", unexpected_specialized_generator + ) + monkeypatch.setattr( + prefill, + "_gen_batch_prefill_independent_paged_module", + unexpected_specialized_generator, + ) + + module = prefill.get_batch_prefill_module("fa3", *_MODULE_ARGS) + shape = (2, 3, 4, 5) + _run_paged( + module, + torch.empty_strided(shape, (60, 20, 5, 1)), + torch.empty_strided(shape, (80, 20, 5, 1)), + ) + assert calls == ["public"] + assert public_spec.build_count == 1 + with pytest.raises(RuntimeError, match="only for standard FA2"): + module.prewarm_paged_kv_stride_variant() + + +class _PrewarmRecorder: + def __init__(self): + self.variants = [] + + def prewarm_paged_kv_stride_variant(self, variant): + self.variants.append(variant) + + +@pytest.mark.parametrize( + "wrapper_cls", + [prefill.BatchPrefillWithPagedKVCacheWrapper, BatchDecodeWithPagedKVCacheWrapper], +) +def test_standard_wrappers_expose_post_plan_prewarm(wrapper_cls): + wrapper = wrapper_cls.__new__(wrapper_cls) + wrapper._plan_info = [1] + wrapper._cached_module = _PrewarmRecorder() + wrapper._jit_module = None + wrapper._backend = "fa2" + wrapper._use_tensor_cores = True + + wrapper.prewarm_paged_kv_stride_variant("independent") + assert wrapper._cached_module.variants == ["independent"] + + wrapper._plan_info = None + with pytest.raises(RuntimeError, match=r"plan\(\) must complete"): + wrapper.prewarm_paged_kv_stride_variant() + + +def test_cascade_wrappers_delegate_prewarm(): + level_one = _PrewarmRecorder() + level_two = _PrewarmRecorder() + multi_level = MultiLevelCascadeAttentionWrapper.__new__( + MultiLevelCascadeAttentionWrapper + ) + multi_level._batch_prefill_wrappers = [level_one, level_two] + multi_level.prewarm_paged_kv_stride_variant("independent") + assert level_one.variants == ["independent"] + assert level_two.variants == ["independent"] + + prefill_child = _PrewarmRecorder() + shared_prefill = BatchPrefillWithSharedPrefixPagedKVCacheWrapper.__new__( + BatchPrefillWithSharedPrefixPagedKVCacheWrapper + ) + shared_prefill._batch_prefill_wrapper = prefill_child + shared_prefill.prewarm_paged_kv_stride_variant("independent") + assert prefill_child.variants == ["independent"] diff --git a/tests/test_helpers/paged_kv.py b/tests/test_helpers/paged_kv.py new file mode 100644 index 00000000000..37bf4f3d01d --- /dev/null +++ b/tests/test_helpers/paged_kv.py @@ -0,0 +1,37 @@ +# Copyright (c) 2026 by FlashInfer team. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Small helpers for constructing deliberate paged-KV stride families.""" + +from __future__ import annotations + +import torch + + +def make_padded_paged_kv_view( + source: torch.Tensor, + kv_layout: str, + *, + padding_heads: int = 2, +) -> torch.Tensor: + """Copy paged KV into an equal-shaped view with different outer strides.""" + head_dimension = {"NHD": 2, "HND": 1}[kv_layout] + shape = list(source.shape) + shape[head_dimension] += padding_heads + view = torch.empty(shape, dtype=source.dtype, device=source.device).narrow( + head_dimension, 0, source.shape[head_dimension] + ) + view.copy_(source) + assert view.stride() != source.stride() + return view