Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
22 changes: 6 additions & 16 deletions csrc/batch_pod.cu
Original file line number Diff line number Diff line change
Expand Up @@ -90,15 +90,10 @@ void batch_pod_with_kv_cache_tensor(
const auto q_stride_n_p = q_p.stride(0);
const auto q_stride_h_p = q_p.stride(1);

// get kv_cache_strides
const int64_t* kv_cache_strides_p = nullptr;
// get independent K/V cache strides
auto k_strides_p = paged_k_cache_p.strides();
auto v_strides_p = paged_v_cache_p.strides();
TVM_FFI_ICHECK_EQ(k_strides_p.size(), v_strides_p.size());
for (int i = 0; i < k_strides_p.size(); ++i) {
TVM_FFI_ICHECK_EQ(k_strides_p[i], v_strides_p[i]);
}
kv_cache_strides_p = k_strides_p.data();

ffi::CUDADeviceGuard device_guard(float_workspace_buffer_p.device().device_id);
const cudaStream_t stream = get_stream(float_workspace_buffer_p.device());
Expand Down Expand Up @@ -141,15 +136,10 @@ void batch_pod_with_kv_cache_tensor(
const auto q_stride_n_d = q_d.stride(0);
const auto q_stride_h_d = q_d.stride(1);

// get kv_cache_strides
const int64_t* kv_cache_strides_d = nullptr;
// get independent K/V cache strides
auto k_strides_d = paged_k_cache_d.strides();
auto v_strides_d = paged_v_cache_d.strides();
TVM_FFI_ICHECK_EQ(k_strides_d.size(), v_strides_d.size());
for (int i = 0; i < k_strides_d.size(); ++i) {
TVM_FFI_ICHECK_EQ(k_strides_d[i], v_strides_d[i]);
}
kv_cache_strides_d = k_strides_d.data();

// Already handled by prefill
// ffi::CUDADeviceGuard device_guard(float_workspace_buffer_d.device().device_id);
Expand All @@ -167,8 +157,8 @@ void batch_pod_with_kv_cache_tensor(
paged_kv_t<DTypeKV, IdType> paged_kv(
num_kv_heads_p, page_size_p, HEAD_DIM_VO, batch_size_p, kv_layout_p,
static_cast<DTypeKV*>(paged_k_cache_p.data_ptr()),
static_cast<DTypeKV*>(paged_v_cache_p.data_ptr()), kv_cache_strides_p,
static_cast<IdType*>(paged_kv_indices_p.data_ptr()),
static_cast<DTypeKV*>(paged_v_cache_p.data_ptr()), k_strides_p.data(),
v_strides_p.data(), static_cast<IdType*>(paged_kv_indices_p.data_ptr()),
static_cast<IdType*>(paged_kv_indptr_p.data_ptr()),
static_cast<IdType*>(paged_kv_last_page_len_p.data_ptr()));
params.paged_kv = paged_kv;
Expand Down Expand Up @@ -245,8 +235,8 @@ void batch_pod_with_kv_cache_tensor(
paged_kv_t<DTypeKV, IdType> paged_kv(
num_kv_heads_d, page_size_d, HEAD_DIM_VO, batch_size_d, kv_layout_d,
static_cast<DTypeKV*>(paged_k_cache_d.data_ptr()),
static_cast<DTypeKV*>(paged_v_cache_d.data_ptr()), kv_cache_strides_d,
static_cast<IdType*>(paged_kv_indices_d.data_ptr()),
static_cast<DTypeKV*>(paged_v_cache_d.data_ptr()), k_strides_d.data(),
v_strides_d.data(), static_cast<IdType*>(paged_kv_indices_d.data_ptr()),
static_cast<IdType*>(paged_kv_indptr_d.data_ptr()),
static_cast<IdType*>(paged_kv_last_page_len_d.data_ptr()));
params.paged_kv = paged_kv;
Expand Down
138 changes: 2 additions & 136 deletions csrc/batch_prefill.cu
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,7 @@

namespace flashinfer {

template <uint32_t CTA_TILE_Q, uint32_t HEAD_DIM_QK, uint32_t HEAD_DIM_VO,
template <bool SAME_KV_STRIDES, uint32_t CTA_TILE_Q, uint32_t HEAD_DIM_QK, uint32_t HEAD_DIM_VO,
PosEncodingMode POS_ENCODING_MODE, bool USE_FP16_QK_REDUCTION, MaskMode MASK_MODE,
typename AttentionVariant, typename Params>
cudaError_t BatchPrefillWithPagedKVCacheDispatched(Params params, typename Params::DTypeO* tmp_v,
Expand Down Expand Up @@ -236,138 +236,4 @@ void BatchPrefillWithRaggedKVCacheRun(TensorView float_workspace_buffer,
});
}

void BatchPrefillWithPagedKVCacheRun(TensorView float_workspace_buffer,
TensorView int_workspace_buffer, Array<int64_t> plan_info_vec,
TensorView q, TensorView paged_k_cache,
TensorView paged_v_cache, TensorView qo_indptr,
TensorView paged_kv_indptr, TensorView paged_kv_indices,
TensorView paged_kv_last_page_len, TensorView o,
Optional<TensorView> maybe_lse, int64_t mask_mode_code,
int64_t layout, int64_t window_left,
bool enable_pdl ADDITIONAL_FUNC_PARAMS) {
PrefillPlanInfo plan_info;
plan_info.FromVector(std::vector<int64_t>(plan_info_vec.begin(), plan_info_vec.end()));
QKVLayout kv_layout = static_cast<QKVLayout>(layout);
int64_t batch_size = paged_kv_indptr.size(0) - 1;
int64_t num_qo_heads = q.size(1);
int64_t num_kv_heads, page_size;
uint32_t head_dim_qk = q.size(2);
if (kv_layout == QKVLayout::kHND) {
num_kv_heads = paged_k_cache.size(1);
page_size = paged_k_cache.size(2);
} else {
page_size = paged_k_cache.size(1);
num_kv_heads = paged_k_cache.size(2);
}

if (maybe_lse) {
const auto& lse = *maybe_lse;
TVM_FFI_ICHECK_EQ(lse.size(0), q.size(0));
TVM_FFI_ICHECK_EQ(lse.size(1), q.size(1));
}

void* float_buffer_ptr = static_cast<void*>(float_workspace_buffer.data_ptr());
void* int_buffer_ptr = static_cast<void*>(int_workspace_buffer.data_ptr());

const MaskMode mask_mode = static_cast<MaskMode>(mask_mode_code);

// get q_stride_n and q_stride_h
const auto q_stride_n = q.stride(0);
const auto q_stride_h = q.stride(1);

// get kv-cache strides
auto k_cache_strides = paged_k_cache.strides();
auto v_cache_strides = paged_v_cache.strides();
TVM_FFI_ICHECK_EQ(paged_k_cache.ndim(), paged_v_cache.ndim());

ffi::CUDADeviceGuard device_guard(float_workspace_buffer.device().device_id);
const cudaStream_t stream = get_stream(float_workspace_buffer.device());

DISPATCH_context(
DTypeQ, DTypeKV, DTypeO, IdType, MASK_MODE, HEAD_DIM_QK, HEAD_DIM_VO, POS_ENCODING_MODE,
USE_SLIDING_WINDOW, USE_LOGITS_SOFT_CAP, USE_FP16_QK_REDUCTION, AttentionVariant,
RaggedParams, PagedParams, [&] {
PagedParams params;

params.q = static_cast<DTypeQ*>(q.data_ptr());
paged_kv_t<DTypeKV, IdType> paged_kv(
num_kv_heads, page_size, HEAD_DIM_VO, batch_size, kv_layout,
static_cast<DTypeKV*>(paged_k_cache.data_ptr()),
static_cast<DTypeKV*>(paged_v_cache.data_ptr()), k_cache_strides.data(),
v_cache_strides.data(), static_cast<IdType*>(paged_kv_indices.data_ptr()),
static_cast<IdType*>(paged_kv_indptr.data_ptr()),
static_cast<IdType*>(paged_kv_last_page_len.data_ptr()));
params.paged_kv = paged_kv;
params.q_indptr = static_cast<IdType*>(qo_indptr.data_ptr());
params.o = static_cast<DTypeO*>(o.data_ptr());

params.lse = maybe_lse ? static_cast<float*>(maybe_lse.value().data_ptr()) : nullptr;
params.num_qo_heads = num_qo_heads;
params.group_size = uint_fastdiv(num_qo_heads / paged_kv.num_heads);
params.q_stride_n = q_stride_n;
params.q_stride_h = q_stride_h;
params.window_left = window_left;

params.request_indices = nullptr;
params.qo_tile_indices = nullptr;
params.kv_tile_indices = nullptr;
params.merge_indptr = nullptr;
params.o_indptr = nullptr;
params.kv_chunk_size_ptr = nullptr;
params.block_valid_mask = nullptr;
params.total_num_rows = nullptr;
params.max_total_num_rows = 0;
params.padded_batch_size = 0;
params.partition_kv = false;
params.k_sf_stride_page = 0;
params.k_sf_stride_n = 0;
params.k_sf_stride_h = 0;
params.v_sf_stride_page = 0;
params.v_sf_stride_n = 0;
params.v_sf_stride_h = 0;

ADDITIONAL_PARAMS_SETTER

DTypeO* tmp_v = nullptr;
float* tmp_s = nullptr;

params.request_indices =
GetPtrFromBaseOffset<IdType>(int_buffer_ptr, plan_info.request_indices_offset);
params.qo_tile_indices =
GetPtrFromBaseOffset<IdType>(int_buffer_ptr, plan_info.qo_tile_indices_offset);
params.kv_tile_indices =
GetPtrFromBaseOffset<IdType>(int_buffer_ptr, plan_info.kv_tile_indices_offset);
params.o_indptr = GetPtrFromBaseOffset<IdType>(int_buffer_ptr, plan_info.o_indptr_offset);
params.kv_chunk_size_ptr =
GetPtrFromBaseOffset<IdType>(int_buffer_ptr, plan_info.kv_chunk_size_ptr_offset);
if (plan_info.split_kv) {
params.merge_indptr =
GetPtrFromBaseOffset<IdType>(int_buffer_ptr, plan_info.merge_indptr_offset);
tmp_v = GetPtrFromBaseOffset<DTypeO>(float_buffer_ptr, plan_info.v_offset);
tmp_s = GetPtrFromBaseOffset<float>(float_buffer_ptr, plan_info.s_offset);
if (plan_info.enable_cuda_graph) {
params.block_valid_mask =
GetPtrFromBaseOffset<bool>(int_buffer_ptr, plan_info.block_valid_mask_offset);
}
}
params.padded_batch_size = plan_info.padded_batch_size;
params.max_total_num_rows = plan_info.total_num_rows;
if (plan_info.enable_cuda_graph) {
params.total_num_rows =
GetPtrFromBaseOffset<uint32_t>(int_buffer_ptr, plan_info.total_num_rows_offset);
}

cudaError_t status = cudaSuccess;

DISPATCH_CTA_TILE_Q(plan_info.cta_tile_q, CTA_TILE_Q, {
status = flashinfer::BatchPrefillWithPagedKVCacheDispatched<
CTA_TILE_Q, HEAD_DIM_QK, HEAD_DIM_VO, POS_ENCODING_MODE,
/*use_fp16_qk_reduction=*/USE_FP16_QK_REDUCTION, MASK_MODE, AttentionVariant,
PagedParams>(params, tmp_v, tmp_s, enable_pdl, stream);
});

TVM_FFI_ICHECK(status == cudaSuccess)
<< "BatchPrefillWithPagedKVCache failed with error " << cudaGetErrorString(status);
return true;
});
}
#include "batch_prefill_paged.cuh"
5 changes: 5 additions & 0 deletions csrc/batch_prefill_customize_config.jinja
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,11 @@ constexpr auto USE_LOGITS_SOFT_CAP = {{ use_logits_soft_cap }};
constexpr auto POS_ENCODING_MODE = {{ pos_encoding_mode }};
constexpr auto USE_SLIDING_WINDOW = {{ use_sliding_window }};

#define PAGED_KV_STRIDE_MODE_RUNTIME 0
#define PAGED_KV_STRIDE_MODE_EQUAL 1
#define PAGED_KV_STRIDE_MODE_INDEPENDENT 2
#define PAGED_KV_STRIDE_MODE PAGED_KV_STRIDE_MODE_{{ paged_kv_stride_mode | upper }}


struct RaggedParams {
using DTypeQ = DTypeQ;
Expand Down
40 changes: 40 additions & 0 deletions csrc/batch_prefill_paged.cu
Original file line number Diff line number Diff line change
@@ -0,0 +1,40 @@
/*
* Copyright (c) 2023 by FlashInfer team.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#include <flashinfer/attention/mask.cuh>
#include <flashinfer/attention/scheduler.cuh>
#include <flashinfer/pos_enc.cuh>

#include "batch_prefill_config.inc"
#include "tvm/ffi/container/array.h"
#include "tvm_ffi_utils.h"

namespace flashinfer {

template <bool SAME_KV_STRIDES, uint32_t CTA_TILE_Q, uint32_t HEAD_DIM_QK, uint32_t HEAD_DIM_VO,
PosEncodingMode POS_ENCODING_MODE, bool USE_FP16_QK_REDUCTION, MaskMode MASK_MODE,
typename AttentionVariant, typename Params>
cudaError_t BatchPrefillWithPagedKVCacheDispatched(Params params, typename Params::DTypeO* tmp_v,
float* tmp_s, bool enable_pdl,
cudaStream_t stream);

} // namespace flashinfer

using namespace flashinfer;

using tvm::ffi::Array;
using tvm::ffi::Optional;

#include "batch_prefill_paged.cuh"
Loading
Loading