Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
33 commits
Select commit Hold shift + click to select a range
db08281
Port PR 601 C128 state mapping fix
zhangxiaolei123456 Jun 18, 2026
b3e3f42
Update PR 601 C128 state mapping optimization
zhangxiaolei123456 Jun 18, 2026
2f4573f
Optimize DSV4 C128 state slot mapping
zhangxiaolei123456 Jun 18, 2026
5fe8555
Remove unused DSV4 SWA translation cache fields
zhangxiaolei123456 Jun 18, 2026
aa17574
Optimize online C128 state pool allocation using request state pool (…
zhangxiaolei123456 Jun 19, 2026
b3ffb37
Merge branch 'main' into sync-pr-601-to-main
zhangxiaolei123456 Jun 19, 2026
263bd41
fix review comments (#604)
zhangxiaolei123456 Jun 20, 2026
699a751
fix review comments and lint (#605)
zhangxiaolei123456 Jun 20, 2026
3c69422
Fix DSV4 backend import lint (#606)
zhangxiaolei123456 Jun 20, 2026
db6d3bb
Fix dsv4 pd state pp slicing (#607)
zhangxiaolei123456 Jun 20, 2026
e878e5d
Merge branch 'main' into sync-pr-601-to-main
zhangxiaolei123456 Jun 22, 2026
0b98a50
Align DSV4 C128 sizing extra slots (#608)
zhangxiaolei123456 Jun 22, 2026
60ba073
Fix dsv4 c128 radix component (#609)
zhangxiaolei123456 Jun 22, 2026
877086c
Fix dsv4 c128 offline pd sync pr 601 (#610)
zhangxiaolei123456 Jun 22, 2026
5bd4905
Fix offline C128 speculative draft cleanup (#611)
zhangxiaolei123456 Jun 22, 2026
fa72841
Optimize offline C128 draft cleanup (#612)
zhangxiaolei123456 Jun 23, 2026
6cf3088
Use logical input length for C128 PD state (#613)
zhangxiaolei123456 Jun 23, 2026
3d0dd24
Clean up prefill transfer input length (#615)
zhangxiaolei123456 Jun 24, 2026
18c78cf
Tighten C128 radix restore live-tail guard (#616)
zhangxiaolei123456 Jun 24, 2026
ed3e5bc
Reset C128 state on radix cache hits (#617)
zhangxiaolei123456 Jun 25, 2026
40a9fea
Merge branch 'main' into sync-pr-601-to-main
zhangxiaolei123456 Jun 25, 2026
bc25dee
Resolve prefill send kv chunk conflict (#619)
zhangxiaolei123456 Jun 25, 2026
5d4e944
Reset C128 state on req slot allocation (#618)
zhangxiaolei123456 Jun 25, 2026
45972ae
fix review comments
zhangxiaolei123456 Jun 26, 2026
a78423f
Update swa_radix_cache.py
zhangxiaolei123456 Jun 26, 2026
76ef819
Merge branch 'main' into sync-pr-601-to-main
zhangxiaolei123456 Jun 26, 2026
f5c0073
Remove C128 req slot allocation reset (#620)
zhangxiaolei123456 Jun 28, 2026
538e60d
Merge branch 'main' into sync-pr-601-to-main
zhangxiaolei123456 Jun 29, 2026
8453104
Merge branch 'main' into sync-pr-601-to-main
zhangxiaolei123456 Jun 29, 2026
bbe2764
Fix online c128 mtp benchmark sync pr 601 (#624)
zhangxiaolei123456 Jun 30, 2026
efa924c
Merge branch 'main' into sync-pr-601-to-main
zhangxiaolei123456 Jun 30, 2026
5b7df61
Merge branch 'main' into sync-pr-601-to-main
zhangxiaolei123456 Jun 30, 2026
4e7a978
Merge branch 'main' into sync-pr-601-to-main
zhangxiaolei123456 Jun 30, 2026
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
32 changes: 2 additions & 30 deletions python/sglang/jit_kernel/csrc/deepseek_v4/c128_online_v2.cuh
Original file line number Diff line number Diff line change
Expand Up @@ -533,9 +533,7 @@ struct OnlineDecodePlanParams {
const int64_t* __restrict__ seq_lens;
const int64_t* __restrict__ req_pool_indices;
const int32_t* __restrict__ req_to_token;
const int64_t* __restrict__ full_to_swa; // (full_cache_size,) int64
int64_t stride_r2t;
int32_t swa_page_size;
int32_t state_slot_offset;
uint32_t batch_size;
};
Expand All @@ -545,10 +543,7 @@ __global__ void plan_c128_online_decode_kernel(const OnlineDecodePlanParams para
if (idx >= params.batch_size) return;
const auto seq_len = static_cast<uint32_t>(params.seq_lens[idx]);
const auto rid = params.req_pool_indices[idx];
const int32_t chunk_start = static_cast<int32_t>((seq_len - 1u) / 128u * 128u);
const int32_t full_loc = params.req_to_token[rid * params.stride_r2t + chunk_start];
const int32_t swa_loc = static_cast<int32_t>(params.full_to_swa[full_loc]);
const int32_t slot = swa_loc / params.swa_page_size + params.state_slot_offset;
const int32_t slot = static_cast<int32_t>(rid) + params.state_slot_offset;
params.plan_d[idx] = DecodePlan{
.seq_len = seq_len,
.write_loc = slot,
Expand All @@ -566,9 +561,7 @@ inline void plan_online_decode(
const tvm::ffi::TensorView seq_lens,
const tvm::ffi::TensorView req_pool_indices,
const tvm::ffi::TensorView req_to_token,
const tvm::ffi::TensorView full_to_swa,
const tvm::ffi::TensorView plan_d_dev_,
const int32_t swa_page_size,
const int32_t state_slot_offset) {
auto B = SymbolicSize{"batch_size"};
auto device_ = SymbolicDevice{};
Expand All @@ -587,15 +580,10 @@ inline void plan_online_decode(
.with_dtype<int32_t>()
.with_device(device_)
.verify(req_to_token);
TensorMatcher({-1}) //
.with_dtype<int64_t>()
.with_device(device_)
.verify(full_to_swa);
TensorMatcher({B, sizeof(DecodePlan)}) //
.with_dtype<uint8_t>()
.with_device(device_)
.verify(plan_d_dev_);
RuntimeCheck(swa_page_size > 0);
RuntimeCheck(state_slot_offset >= 0);

const auto batch_size = static_cast<uint32_t>(B.unwrap());
Expand All @@ -610,9 +598,7 @@ inline void plan_online_decode(
.seq_lens = static_cast<const int64_t*>(seq_lens.data_ptr()),
.req_pool_indices = static_cast<const int64_t*>(req_pool_indices.data_ptr()),
.req_to_token = static_cast<const int32_t*>(req_to_token.data_ptr()),
.full_to_swa = static_cast<const int64_t*>(full_to_swa.data_ptr()),
.stride_r2t = stride_r2t,
.swa_page_size = swa_page_size,
.state_slot_offset = state_slot_offset,
.batch_size = batch_size,
};
Expand Down Expand Up @@ -685,9 +671,7 @@ struct OnlinePrefillStage1Params {
CompressPlan* __restrict__ plan_w;
const int64_t* __restrict__ req_pool_indices; // (batch_size,)
const int32_t* __restrict__ req_to_token; // (num_reqs, max_tokens)
const int64_t* __restrict__ full_to_swa; // (full_cache_size,)
int64_t stride_r2t;
int32_t swa_page_size;
int32_t state_slot_offset;
uint32_t num_c;
uint32_t num_w;
Expand All @@ -704,11 +688,7 @@ __global__ void plan_c128_online_prefill_kernel(const OnlinePrefillStage1Params
if (plan.is_invalid()) return;
const auto batch_id = plan.read_page_0;
const auto rid = params.req_pool_indices[batch_id];
const int32_t position = static_cast<int32_t>(plan.seq_len - 1u);
const int32_t chunk_start = (position / 128) * 128;
const int32_t full_loc = params.req_to_token[rid * params.stride_r2t + chunk_start];
const int32_t swa_loc = static_cast<int32_t>(params.full_to_swa[full_loc]);
const int32_t main_slot = swa_loc / params.swa_page_size;
const int32_t main_slot = static_cast<int32_t>(rid);
plan.read_page_0 = main_slot + params.state_slot_offset;
plan.read_page_1 = main_slot;
*plan_ptr = plan;
Expand All @@ -721,12 +701,10 @@ inline OnlinePrefillPlan plan_online_prefill(
const tvm::ffi::TensorView extend_lens,
const tvm::ffi::TensorView req_pool_indices,
const tvm::ffi::TensorView req_to_token,
const tvm::ffi::TensorView full_to_swa,
const tvm::ffi::TensorView plan_c_pin,
const tvm::ffi::TensorView plan_w_pin,
const tvm::ffi::TensorView plan_c_dev_,
const tvm::ffi::TensorView plan_w_dev_,
const int32_t swa_page_size,
const int32_t state_slot_offset,
const bool use_cuda_graph) {
auto B = SymbolicSize{"batch_size"};
Expand All @@ -749,10 +727,6 @@ inline OnlinePrefillPlan plan_online_prefill(
.with_dtype<int32_t>()
.with_device(device_)
.verify(req_to_token);
TensorMatcher({-1}) //
.with_dtype<int64_t>()
.with_device(device_)
.verify(full_to_swa);
TensorMatcher({N, sizeof(CompressPlan)}) //
.with_dtype<uint8_t>()
.with_device(cpu)
Expand Down Expand Up @@ -878,9 +852,7 @@ inline OnlinePrefillPlan plan_online_prefill(
.plan_w = plan_w_dev_ptr,
.req_pool_indices = static_cast<const int64_t*>(req_pool_indices.data_ptr()),
.req_to_token = static_cast<const int32_t*>(req_to_token.data_ptr()),
.full_to_swa = static_cast<const int64_t*>(full_to_swa.data_ptr()),
.stride_r2t = req_to_token.stride(0),
.swa_page_size = swa_page_size,
.state_slot_offset = state_slot_offset,
.num_c = num_c_padded,
.num_w = num_w_padded,
Expand Down
81 changes: 53 additions & 28 deletions python/sglang/jit_kernel/csrc/deepseek_v4/c_plan.cuh
Original file line number Diff line number Diff line change
Expand Up @@ -53,7 +53,7 @@ struct Prefill1Params {
PlanW* plan_w;
const RID_T* rid_ptr; // [batch_size]
const R2T_T* r2t_ptr; // [num_reqs, stride_r2t]
const F2S_T* f2s_ptr; // [num_swa_slots]
const F2S_T* f2s_ptr; // [num_full_slots], full_loc -> swa_loc
int64_t stride_r2t;
uint32_t num_c;
uint32_t num_w;
Expand All @@ -69,7 +69,7 @@ struct DecodeParams {
PlanD* plan_d;
const RID_T* rid_ptr; // [batch_size]
const R2T_T* r2t_ptr; // [num_reqs, stride_r2t]
const F2S_T* f2s_ptr; // [num_swa_slots]
const F2S_T* f2s_ptr; // [num_full_slots], full_loc -> swa_loc
const IDX_T* seq_ptr; // [batch_size]
int64_t stride_r2t;
uint32_t batch_size;
Expand Down Expand Up @@ -297,6 +297,9 @@ __global__ void plan_compress_prefill_kernel_1(const Prefill1Params params) {
const auto ring_offset = swa_loc % params.ring_size;
return swa_page * params.ring_size + ring_offset;
};
const auto compute_c128_loc = [&](int64_t rid, int32_t position) {
return static_cast<int32_t>(rid * params.ring_size + position % params.ring_size);
};

if (!plan_c.is_invalid()) { // 1. in bound. 2. not masked
if (plan_c.buffer_len > 0) {
Expand All @@ -307,12 +310,17 @@ __global__ void plan_compress_prefill_kernel_1(const Prefill1Params params) {
const auto position_1 = static_cast<int32_t>(plan_c.seq_len - 1);
// only used for c4, harmless for c128
const auto position_0 = max(position_1 - params.compress_ratio, 0);
const auto raw_loc_0 = mapping[position_0];
const auto raw_loc_1 = mapping[position_1];
const auto swa_loc_0 = params.f2s_ptr[raw_loc_0];
const auto swa_loc_1 = params.f2s_ptr[raw_loc_1];
plan_c.read_page_0 = compute_loc(swa_loc_0) / params.compress_ratio;
plan_c.read_page_1 = compute_loc(swa_loc_1) / params.compress_ratio;
if (params.compress_ratio == 128) {
plan_c.read_page_0 = compute_c128_loc(rid, position_0) / 128;
plan_c.read_page_1 = compute_c128_loc(rid, position_1) / 128;
} else {
const auto raw_loc_0 = mapping[position_0];
const auto raw_loc_1 = mapping[position_1];
const auto state_loc_0 = params.f2s_ptr[raw_loc_0];
const auto state_loc_1 = params.f2s_ptr[raw_loc_1];
plan_c.read_page_0 = compute_loc(state_loc_0) / params.compress_ratio;
plan_c.read_page_1 = compute_loc(state_loc_1) / params.compress_ratio;
}
params.plan_c[idx] = plan_c;
}
} else if (idx < params.num_c_padded) {
Expand All @@ -325,10 +333,13 @@ __global__ void plan_compress_prefill_kernel_1(const Prefill1Params params) {
const auto mapping = params.r2t_ptr + rid * params.stride_r2t;
// `seq_len` (`write_loc`) may not be aligned here
const auto position = static_cast<int32_t>(plan_w.write_loc - 1);
const auto raw_loc = mapping[position];
const auto swa_loc = params.f2s_ptr[raw_loc];
plan_w.ragged_id = ragged_id;
plan_w.write_loc = compute_loc(swa_loc);
if (params.compress_ratio == 128) {
plan_w.write_loc = compute_c128_loc(rid, position);
} else {
const auto raw_loc = mapping[position];
plan_w.write_loc = compute_loc(params.f2s_ptr[raw_loc]);
}
params.plan_w[idx] = plan_w;
} else if (idx < params.num_w_padded) {
params.plan_w[idx] = PlanW::invalid();
Expand All @@ -345,16 +356,28 @@ __global__ void plan_compress_decode_kernel(const DecodeParams params) {
const auto ring_offset = swa_loc % params.ring_size;
return swa_page * params.ring_size + ring_offset;
};
const auto compute_c128_loc = [&](int64_t rid, int32_t position) {
return static_cast<int32_t>(rid * params.ring_size + position % params.ring_size);
};
const auto seq_len = static_cast<int32_t>(params.seq_ptr[idx]);
const auto position_1 = static_cast<int32_t>(seq_len - 1);
const auto position_0 = max(position_1 - params.compress_ratio, 0);
const auto raw_loc_0 = mapping[position_0];
const auto raw_loc_1 = mapping[position_1];
const auto swa_loc_0 = params.f2s_ptr[raw_loc_0];
const auto swa_loc_1 = params.f2s_ptr[raw_loc_1];
const auto write_loc = compute_loc(swa_loc_1);
const auto read_page_0 = compute_loc(swa_loc_0) / params.compress_ratio;
const auto read_page_1 = write_loc / params.compress_ratio;
int32_t write_loc;
int32_t read_page_0;
int32_t read_page_1;
if (params.compress_ratio == 128) {
write_loc = compute_c128_loc(rid, position_1);
read_page_0 = compute_c128_loc(rid, position_0) / 128;
read_page_1 = compute_c128_loc(rid, position_1) / 128;
} else {
const auto raw_loc_0 = mapping[position_0];
const auto raw_loc_1 = mapping[position_1];
const auto state_loc_0 = params.f2s_ptr[raw_loc_0];
const auto state_loc_1 = params.f2s_ptr[raw_loc_1];
write_loc = static_cast<int32_t>(compute_loc(state_loc_1));
read_page_0 = static_cast<int32_t>(compute_loc(state_loc_0) / params.compress_ratio);
read_page_1 = static_cast<int32_t>(write_loc / params.compress_ratio);
}
params.plan_d[idx] = {
.seq_len = static_cast<uint32_t>(seq_len),
.write_loc = write_loc,
Expand Down Expand Up @@ -425,9 +448,9 @@ __global__ void plan_compress_decode_legacy_kernel(const DecodeParamsLegacy para
const auto seq_len = static_cast<int32_t>(params.seq_ptr[idx]);
const auto position_1 = seq_len - 1;
const auto position_0 = max(position_1 - params.compress_ratio, 0);
const auto write_loc = legacy_compute_loc(rid, position_1);
const auto read_page_0 = legacy_compute_page(rid, position_0);
const auto read_page_1 = legacy_compute_page(rid, position_1);
const int32_t write_loc = legacy_compute_loc(rid, position_1);
const int32_t read_page_0 = legacy_compute_page(rid, position_0);
const int32_t read_page_1 = legacy_compute_page(rid, position_1);
params.plan_d[idx] = {
.seq_len = static_cast<uint32_t>(seq_len),
.write_loc = write_loc,
Expand All @@ -443,7 +466,9 @@ using PrefillPlan = tvm::ffi::Tuple<tvm::ffi::Tensor, tvm::ffi::Tensor>;
* Inputs (all CPU-resident):
* @param req_pool_indices `[batch_size]` int64_t
* @param req_to_token `[num_reqs, max_tokens_per_req]` int64_t
* @param full_to_swa `[num_swa_slots]` int64_t
* @param full_to_state `[full_cache_size]` int64_t. For c4 this maps
* full loc -> SWA loc; ignored for c128, whose
* state slot is request-scoped.
* @param seq_lens `[batch_size]` int64
* @param extend_lens `[batch_size]` int64
* @param compress_plan `[num_q_tokens, 16]` uint8 (output)
Expand All @@ -455,7 +480,7 @@ using PrefillPlan = tvm::ffi::Tuple<tvm::ffi::Tensor, tvm::ffi::Tensor>;
inline PrefillPlan plan_compress_prefill(
const tvm::ffi::TensorView req_pool_indices, // GPU
const tvm::ffi::TensorView req_to_token, // GPU
const tvm::ffi::TensorView full_to_swa, // GPU
const tvm::ffi::TensorView full_to_state, // GPU
const tvm::ffi::TensorView seq_lens, // CPU/GPU
const tvm::ffi::TensorView extend_lens, // CPU/GPU
const tvm::ffi::TensorView pin_buffer, // CPU
Expand All @@ -482,7 +507,7 @@ inline PrefillPlan plan_compress_prefill(
TensorMatcher({-1}) //
.with_dtype<F2S_T>()
.with_device(device_)
.verify(full_to_swa);
.verify(full_to_state);
TensorMatcher({B}) //
.with_dtype<IDX_T>()
.with_device(cpu_or_gpu)
Expand All @@ -500,7 +525,7 @@ inline PrefillPlan plan_compress_prefill(
const auto ext_ptr = static_cast<const IDX_T*>(extend_lens.data_ptr());
const auto rid_ptr = static_cast<const RID_T*>(req_pool_indices.data_ptr());
const auto r2t_ptr = static_cast<const R2T_T*>(req_to_token.data_ptr());
const auto f2s_ptr = static_cast<const F2S_T*>(full_to_swa.data_ptr());
const auto f2s_ptr = static_cast<const F2S_T*>(full_to_state.data_ptr());

const auto batch_size = static_cast<uint32_t>(B.unwrap());
constexpr auto kMaxTokens = static_cast<uint32_t>(std::numeric_limits<uint16_t>::max());
Expand Down Expand Up @@ -637,7 +662,7 @@ inline PrefillPlan plan_compress_prefill(
inline tvm::ffi::Tensor plan_compress_decode(
const tvm::ffi::TensorView req_pool_indices, // GPU
const tvm::ffi::TensorView req_to_token, // GPU
const tvm::ffi::TensorView full_to_swa, // GPU
const tvm::ffi::TensorView full_to_state, // GPU
const tvm::ffi::TensorView seq_lens, // CPU/GPU
const int32_t compress_ratio,
const int32_t swa_page_size,
Expand All @@ -657,7 +682,7 @@ inline tvm::ffi::Tensor plan_compress_decode(
TensorMatcher({-1}) //
.with_dtype<F2S_T>()
.with_device(device_)
.verify(full_to_swa);
.verify(full_to_state);
TensorMatcher({B}) //
.with_dtype<IDX_T>()
.with_device(device_)
Expand All @@ -670,7 +695,7 @@ inline tvm::ffi::Tensor plan_compress_decode(
.plan_d = static_cast<PlanD*>(D.data_ptr()),
.rid_ptr = static_cast<const RID_T*>(req_pool_indices.data_ptr()),
.r2t_ptr = static_cast<const R2T_T*>(req_to_token.data_ptr()),
.f2s_ptr = static_cast<const F2S_T*>(full_to_swa.data_ptr()),
.f2s_ptr = static_cast<const F2S_T*>(full_to_state.data_ptr()),
.seq_ptr = static_cast<const IDX_T*>(seq_lens.data_ptr()),
.stride_r2t = req_to_token.size(1),
.batch_size = batch_size,
Expand Down
Loading
Loading