From db08281789e6648aeca59604792b213edde28b76 Mon Sep 17 00:00:00 2001 From: zhangxiaolei Date: Thu, 18 Jun 2026 12:40:33 +0800 Subject: [PATCH 01/24] Port PR 601 C128 state mapping fix --- .../csrc/deepseek_v4/c128_online_v2.cuh | 29 ++-- .../jit_kernel/csrc/deepseek_v4/c_plan.cuh | 61 ++++--- .../csrc/deepseek_v4/online_c128_mtp.cuh | 152 ++++-------------- python/sglang/jit_kernel/dsv4/compress.py | 22 ++- .../sglang/jit_kernel/dsv4/online_c128_mtp.py | 6 +- .../jit_kernel/tests/deepseek_v4/common.py | 4 +- .../layers/attention/dsv4/compressor_v2.py | 20 +-- python/sglang/srt/mem_cache/allocator/swa.py | 17 ++ .../srt/mem_cache/deepseek_v4_memory_pool.py | 53 ++++++ .../model_runner_kv_cache_mixin.py | 2 +- .../srt/model_executor/pool_configurator.py | 14 +- 11 files changed, 179 insertions(+), 201 deletions(-) diff --git a/python/sglang/jit_kernel/csrc/deepseek_v4/c128_online_v2.cuh b/python/sglang/jit_kernel/csrc/deepseek_v4/c128_online_v2.cuh index 02f2630780fc..d7209960b8d5 100644 --- a/python/sglang/jit_kernel/csrc/deepseek_v4/c128_online_v2.cuh +++ b/python/sglang/jit_kernel/csrc/deepseek_v4/c128_online_v2.cuh @@ -533,9 +533,8 @@ struct OnlineDecodePlanParams { const int64_t* __restrict__ seq_lens; const int64_t* __restrict__ req_pool_indices; const int32_t* __restrict__ req_to_token; - const int64_t* __restrict__ full_to_swa; // (full_cache_size,) int64 + const int64_t* __restrict__ full_to_c128_state; // (full_cache_size,) int64 int64_t stride_r2t; - int32_t swa_page_size; int32_t state_slot_offset; uint32_t batch_size; }; @@ -547,8 +546,7 @@ __global__ void plan_c128_online_decode_kernel(const OnlineDecodePlanParams para const auto rid = params.req_pool_indices[idx]; const int32_t chunk_start = static_cast((seq_len - 1u) / 128u * 128u); const int32_t full_loc = params.req_to_token[rid * params.stride_r2t + chunk_start]; - const int32_t swa_loc = static_cast(params.full_to_swa[full_loc]); - const int32_t slot = swa_loc / params.swa_page_size + params.state_slot_offset; + const int32_t slot = static_cast(params.full_to_c128_state[full_loc]) + params.state_slot_offset; params.plan_d[idx] = DecodePlan{ .seq_len = seq_len, .write_loc = slot, @@ -566,9 +564,8 @@ inline void plan_online_decode( const tvm::ffi::TensorView seq_lens, const tvm::ffi::TensorView req_pool_indices, const tvm::ffi::TensorView req_to_token, - const tvm::ffi::TensorView full_to_swa, + const tvm::ffi::TensorView full_to_c128_state, const tvm::ffi::TensorView plan_d_dev_, - const int32_t swa_page_size, const int32_t state_slot_offset) { auto B = SymbolicSize{"batch_size"}; auto device_ = SymbolicDevice{}; @@ -590,12 +587,11 @@ inline void plan_online_decode( TensorMatcher({-1}) // .with_dtype() .with_device(device_) - .verify(full_to_swa); + .verify(full_to_c128_state); TensorMatcher({B, sizeof(DecodePlan)}) // .with_dtype() .with_device(device_) .verify(plan_d_dev_); - RuntimeCheck(swa_page_size > 0); RuntimeCheck(state_slot_offset >= 0); const auto batch_size = static_cast(B.unwrap()); @@ -610,9 +606,8 @@ inline void plan_online_decode( .seq_lens = static_cast(seq_lens.data_ptr()), .req_pool_indices = static_cast(req_pool_indices.data_ptr()), .req_to_token = static_cast(req_to_token.data_ptr()), - .full_to_swa = static_cast(full_to_swa.data_ptr()), + .full_to_c128_state = static_cast(full_to_c128_state.data_ptr()), .stride_r2t = stride_r2t, - .swa_page_size = swa_page_size, .state_slot_offset = state_slot_offset, .batch_size = batch_size, }; @@ -685,9 +680,8 @@ struct OnlinePrefillStage1Params { CompressPlan* __restrict__ plan_w; const int64_t* __restrict__ req_pool_indices; // (batch_size,) const int32_t* __restrict__ req_to_token; // (num_reqs, max_tokens) - const int64_t* __restrict__ full_to_swa; // (full_cache_size,) + const int64_t* __restrict__ full_to_c128_state; // (full_cache_size,) int64_t stride_r2t; - int32_t swa_page_size; int32_t state_slot_offset; uint32_t num_c; uint32_t num_w; @@ -707,8 +701,7 @@ __global__ void plan_c128_online_prefill_kernel(const OnlinePrefillStage1Params const int32_t position = static_cast(plan.seq_len - 1u); const int32_t chunk_start = (position / 128) * 128; const int32_t full_loc = params.req_to_token[rid * params.stride_r2t + chunk_start]; - const int32_t swa_loc = static_cast(params.full_to_swa[full_loc]); - const int32_t main_slot = swa_loc / params.swa_page_size; + const int32_t main_slot = static_cast(params.full_to_c128_state[full_loc]); plan.read_page_0 = main_slot + params.state_slot_offset; plan.read_page_1 = main_slot; *plan_ptr = plan; @@ -721,12 +714,11 @@ inline OnlinePrefillPlan plan_online_prefill( const tvm::ffi::TensorView extend_lens, const tvm::ffi::TensorView req_pool_indices, const tvm::ffi::TensorView req_to_token, - const tvm::ffi::TensorView full_to_swa, + const tvm::ffi::TensorView full_to_c128_state, const tvm::ffi::TensorView plan_c_pin, const tvm::ffi::TensorView plan_w_pin, const tvm::ffi::TensorView plan_c_dev_, const tvm::ffi::TensorView plan_w_dev_, - const int32_t swa_page_size, const int32_t state_slot_offset, const bool use_cuda_graph) { auto B = SymbolicSize{"batch_size"}; @@ -752,7 +744,7 @@ inline OnlinePrefillPlan plan_online_prefill( TensorMatcher({-1}) // .with_dtype() .with_device(device_) - .verify(full_to_swa); + .verify(full_to_c128_state); TensorMatcher({N, sizeof(CompressPlan)}) // .with_dtype() .with_device(cpu) @@ -878,9 +870,8 @@ inline OnlinePrefillPlan plan_online_prefill( .plan_w = plan_w_dev_ptr, .req_pool_indices = static_cast(req_pool_indices.data_ptr()), .req_to_token = static_cast(req_to_token.data_ptr()), - .full_to_swa = static_cast(full_to_swa.data_ptr()), + .full_to_c128_state = static_cast(full_to_c128_state.data_ptr()), .stride_r2t = req_to_token.stride(0), - .swa_page_size = swa_page_size, .state_slot_offset = state_slot_offset, .num_c = num_c_padded, .num_w = num_w_padded, diff --git a/python/sglang/jit_kernel/csrc/deepseek_v4/c_plan.cuh b/python/sglang/jit_kernel/csrc/deepseek_v4/c_plan.cuh index 4e2f2ed287db..7336a2b0d26c 100644 --- a/python/sglang/jit_kernel/csrc/deepseek_v4/c_plan.cuh +++ b/python/sglang/jit_kernel/csrc/deepseek_v4/c_plan.cuh @@ -53,7 +53,7 @@ struct Prefill1Params { PlanW* plan_w; const RID_T* rid_ptr; // [batch_size] const R2T_T* r2t_ptr; // [num_reqs, stride_r2t] - const F2S_T* f2s_ptr; // [num_swa_slots] + const F2S_T* state_map_ptr; // full_loc -> SWA loc for c4, C128 state loc for c128 int64_t stride_r2t; uint32_t num_c; uint32_t num_w; @@ -69,7 +69,7 @@ struct DecodeParams { PlanD* plan_d; const RID_T* rid_ptr; // [batch_size] const R2T_T* r2t_ptr; // [num_reqs, stride_r2t] - const F2S_T* f2s_ptr; // [num_swa_slots] + const F2S_T* state_map_ptr; // full_loc -> SWA loc for c4, C128 state loc for c128 const IDX_T* seq_ptr; // [batch_size] int64_t stride_r2t; uint32_t batch_size; @@ -309,10 +309,15 @@ __global__ void plan_compress_prefill_kernel_1(const Prefill1Params params) { const auto position_0 = max(position_1 - params.compress_ratio, 0); const auto raw_loc_0 = mapping[position_0]; const auto raw_loc_1 = mapping[position_1]; - const auto swa_loc_0 = params.f2s_ptr[raw_loc_0]; - const auto swa_loc_1 = params.f2s_ptr[raw_loc_1]; - plan_c.read_page_0 = compute_loc(swa_loc_0) / params.compress_ratio; - plan_c.read_page_1 = compute_loc(swa_loc_1) / params.compress_ratio; + const auto state_loc_0 = params.state_map_ptr[raw_loc_0]; + const auto state_loc_1 = params.state_map_ptr[raw_loc_1]; + if (params.compress_ratio == 128) { + plan_c.read_page_0 = state_loc_0 / 128; + plan_c.read_page_1 = state_loc_1 / 128; + } else { + plan_c.read_page_0 = compute_loc(state_loc_0) / params.compress_ratio; + plan_c.read_page_1 = compute_loc(state_loc_1) / params.compress_ratio; + } params.plan_c[idx] = plan_c; } } else if (idx < params.num_c_padded) { @@ -326,9 +331,10 @@ __global__ void plan_compress_prefill_kernel_1(const Prefill1Params params) { // `seq_len` (`write_loc`) may not be aligned here const auto position = static_cast(plan_w.write_loc - 1); const auto raw_loc = mapping[position]; - const auto swa_loc = params.f2s_ptr[raw_loc]; + const auto state_loc = params.state_map_ptr[raw_loc]; plan_w.ragged_id = ragged_id; - plan_w.write_loc = compute_loc(swa_loc); + plan_w.write_loc = + params.compress_ratio == 128 ? state_loc : compute_loc(state_loc); params.plan_w[idx] = plan_w; } else if (idx < params.num_w_padded) { params.plan_w[idx] = PlanW::invalid(); @@ -350,11 +356,14 @@ __global__ void plan_compress_decode_kernel(const DecodeParams params) { const auto position_0 = max(position_1 - params.compress_ratio, 0); const auto raw_loc_0 = mapping[position_0]; const auto raw_loc_1 = mapping[position_1]; - const auto swa_loc_0 = params.f2s_ptr[raw_loc_0]; - const auto swa_loc_1 = params.f2s_ptr[raw_loc_1]; - const auto write_loc = compute_loc(swa_loc_1); - const auto read_page_0 = compute_loc(swa_loc_0) / params.compress_ratio; - const auto read_page_1 = write_loc / params.compress_ratio; + const auto state_loc_0 = params.state_map_ptr[raw_loc_0]; + const auto state_loc_1 = params.state_map_ptr[raw_loc_1]; + const int32_t write_loc = static_cast( + params.compress_ratio == 128 ? state_loc_1 : compute_loc(state_loc_1)); + const int32_t read_page_0 = static_cast( + params.compress_ratio == 128 ? state_loc_0 / 128 : compute_loc(state_loc_0) / params.compress_ratio); + const int32_t read_page_1 = static_cast( + params.compress_ratio == 128 ? state_loc_1 / 128 : write_loc / params.compress_ratio); params.plan_d[idx] = { .seq_len = static_cast(seq_len), .write_loc = write_loc, @@ -425,9 +434,9 @@ __global__ void plan_compress_decode_legacy_kernel(const DecodeParamsLegacy para const auto seq_len = static_cast(params.seq_ptr[idx]); const auto position_1 = seq_len - 1; const auto position_0 = max(position_1 - params.compress_ratio, 0); - const auto write_loc = legacy_compute_loc(rid, position_1); - const auto read_page_0 = legacy_compute_page(rid, position_0); - const auto read_page_1 = legacy_compute_page(rid, position_1); + const int32_t write_loc = legacy_compute_loc(rid, position_1); + const int32_t read_page_0 = legacy_compute_page(rid, position_0); + const int32_t read_page_1 = legacy_compute_page(rid, position_1); params.plan_d[idx] = { .seq_len = static_cast(seq_len), .write_loc = write_loc, @@ -443,7 +452,9 @@ using PrefillPlan = tvm::ffi::Tuple; * Inputs (all CPU-resident): * @param req_pool_indices `[batch_size]` int64_t * @param req_to_token `[num_reqs, max_tokens_per_req]` int64_t - * @param full_to_swa `[num_swa_slots]` int64_t + * @param full_to_state `[full_cache_size]` int64_t. For c4 this maps + * full loc -> SWA loc; for c128 it maps full loc -> + * C128 state loc. * @param seq_lens `[batch_size]` int64 * @param extend_lens `[batch_size]` int64 * @param compress_plan `[num_q_tokens, 16]` uint8 (output) @@ -455,7 +466,7 @@ using PrefillPlan = tvm::ffi::Tuple; inline PrefillPlan plan_compress_prefill( const tvm::ffi::TensorView req_pool_indices, // GPU const tvm::ffi::TensorView req_to_token, // GPU - const tvm::ffi::TensorView full_to_swa, // GPU + const tvm::ffi::TensorView full_to_state, // GPU const tvm::ffi::TensorView seq_lens, // CPU/GPU const tvm::ffi::TensorView extend_lens, // CPU/GPU const tvm::ffi::TensorView pin_buffer, // CPU @@ -482,7 +493,7 @@ inline PrefillPlan plan_compress_prefill( TensorMatcher({-1}) // .with_dtype() .with_device(device_) - .verify(full_to_swa); + .verify(full_to_state); TensorMatcher({B}) // .with_dtype() .with_device(cpu_or_gpu) @@ -500,7 +511,7 @@ inline PrefillPlan plan_compress_prefill( const auto ext_ptr = static_cast(extend_lens.data_ptr()); const auto rid_ptr = static_cast(req_pool_indices.data_ptr()); const auto r2t_ptr = static_cast(req_to_token.data_ptr()); - const auto f2s_ptr = static_cast(full_to_swa.data_ptr()); + const auto state_map_ptr = static_cast(full_to_state.data_ptr()); const auto batch_size = static_cast(B.unwrap()); constexpr auto kMaxTokens = static_cast(std::numeric_limits::max()); @@ -541,7 +552,7 @@ inline PrefillPlan plan_compress_prefill( .plan_w = static_cast(W.data_ptr()), .rid_ptr = rid_ptr, .r2t_ptr = r2t_ptr, - .f2s_ptr = f2s_ptr, + .state_map_ptr = state_map_ptr, .stride_r2t = req_to_token.stride(0), .num_c = num_q_tokens, .num_w = num_q_tokens, @@ -617,7 +628,7 @@ inline PrefillPlan plan_compress_prefill( .plan_w = static_cast(W.data_ptr()), .rid_ptr = rid_ptr, .r2t_ptr = r2t_ptr, - .f2s_ptr = f2s_ptr, + .state_map_ptr = state_map_ptr, .stride_r2t = req_to_token.size(1), .num_c = counter_c, .num_w = counter_w, @@ -637,7 +648,7 @@ inline PrefillPlan plan_compress_prefill( inline tvm::ffi::Tensor plan_compress_decode( const tvm::ffi::TensorView req_pool_indices, // GPU const tvm::ffi::TensorView req_to_token, // GPU - const tvm::ffi::TensorView full_to_swa, // GPU + const tvm::ffi::TensorView full_to_state, // GPU const tvm::ffi::TensorView seq_lens, // CPU/GPU const int32_t compress_ratio, const int32_t swa_page_size, @@ -657,7 +668,7 @@ inline tvm::ffi::Tensor plan_compress_decode( TensorMatcher({-1}) // .with_dtype() .with_device(device_) - .verify(full_to_swa); + .verify(full_to_state); TensorMatcher({B}) // .with_dtype() .with_device(device_) @@ -670,7 +681,7 @@ inline tvm::ffi::Tensor plan_compress_decode( .plan_d = static_cast(D.data_ptr()), .rid_ptr = static_cast(req_pool_indices.data_ptr()), .r2t_ptr = static_cast(req_to_token.data_ptr()), - .f2s_ptr = static_cast(full_to_swa.data_ptr()), + .state_map_ptr = static_cast(full_to_state.data_ptr()), .seq_ptr = static_cast(seq_lens.data_ptr()), .stride_r2t = req_to_token.size(1), .batch_size = batch_size, diff --git a/python/sglang/jit_kernel/csrc/deepseek_v4/online_c128_mtp.cuh b/python/sglang/jit_kernel/csrc/deepseek_v4/online_c128_mtp.cuh index db1b94d89344..c28ae49f456e 100644 --- a/python/sglang/jit_kernel/csrc/deepseek_v4/online_c128_mtp.cuh +++ b/python/sglang/jit_kernel/csrc/deepseek_v4/online_c128_mtp.cuh @@ -23,7 +23,7 @@ struct OnlineC128MTPWritePrefixParams { const TSeq* __restrict__ seq_lens; const TReq* __restrict__ req_pool_indices; const int32_t* __restrict__ req_to_token; - const int64_t* __restrict__ full_to_swa; + const int64_t* __restrict__ full_to_c128_state; const float* __restrict__ ape; float* __restrict__ state; int64_t kv_score_stride_b; @@ -31,7 +31,6 @@ struct OnlineC128MTPWritePrefixParams { int64_t ape_stride_r; int64_t state_stride_b; int64_t layer_bs; - int64_t swa_page_size; int64_t num_verify_tokens; int64_t state_slot_stride; }; @@ -50,13 +49,12 @@ struct OnlineC128MTPCommitPendingParams { const TSeq* __restrict__ cur_seq_lens; const TReq* __restrict__ cur_req_pool_indices; const int32_t* __restrict__ req_to_token; - const int64_t* __restrict__ full_to_swa; + const int64_t* __restrict__ full_to_c128_state; const int64_t* __restrict__ pending_seq_lens; float* __restrict__ state; int64_t cur_bs; int64_t req_to_token_stride_b; int64_t state_stride_b; - int64_t swa_page_size; int64_t num_verify_tokens; int64_t state_slot_stride; int64_t max_num_reqs; @@ -95,9 +93,9 @@ __global__ void online_c128_mtp_commit_pending_kernel(const OnlineC128MTPCommitP if ((final_seq & 127) == 0) return; const int64_t chunk_start = ((final_seq - 1) / 128) * 128; - const int64_t full_loc = static_cast(params.req_to_token[req * params.req_to_token_stride_b + chunk_start]); - const int64_t swa_loc = params.full_to_swa[full_loc]; - const int64_t slot = swa_loc / params.swa_page_size; + const int64_t full_loc = + static_cast(params.req_to_token[req * params.req_to_token_stride_b + chunk_start]); + const int64_t slot = params.full_to_c128_state[full_loc]; const float* const src = params.state + (slot + accept * params.state_slot_stride) * params.state_stride_b; float* const dst = params.state + slot * params.state_stride_b; @@ -121,8 +119,7 @@ __global__ void online_c128_mtp_write_prefix_kernel(const OnlineC128MTPWritePref const int64_t chunk_start = ((seq_before - 1) / 128) * 128; const int64_t full_loc = static_cast(params.req_to_token[req_idx * params.req_to_token_stride_b + chunk_start]); - const int64_t swa_loc = params.full_to_swa[full_loc]; - init_slot = swa_loc / params.swa_page_size; + init_slot = params.full_to_c128_state[full_loc]; } const int64_t d = static_cast(threadIdx.x); @@ -176,8 +173,7 @@ __global__ void online_c128_mtp_write_prefix_kernel(const OnlineC128MTPWritePref const int64_t chunk_start = ((final_seq - 1) / 128) * 128; const int64_t full_loc = static_cast(params.req_to_token[req_idx * params.req_to_token_stride_b + chunk_start]); - const int64_t swa_loc = params.full_to_swa[full_loc]; - const int64_t slot = swa_loc / params.swa_page_size + (step + 1) * params.state_slot_stride; + const int64_t slot = params.full_to_c128_state[full_loc] + (step + 1) * params.state_slot_stride; float* const out = params.state + slot * params.state_stride_b; out[d] = run_max; out[kHeadDim + d] = run_sum; @@ -200,11 +196,10 @@ struct OnlineC128MTPWritePrefixKernel { tvm::ffi::TensorView seq_lens, tvm::ffi::TensorView req_pool_indices, tvm::ffi::TensorView req_to_token, - tvm::ffi::TensorView full_to_swa, + tvm::ffi::TensorView full_to_c128_state, tvm::ffi::TensorView ape, tvm::ffi::TensorView state, int64_t layer_bs, - int64_t swa_page_size, int64_t num_verify_tokens, int64_t state_slot_stride, DLDevice device) { @@ -215,7 +210,7 @@ struct OnlineC128MTPWritePrefixKernel { .seq_lens = static_cast(seq_lens.data_ptr()), .req_pool_indices = static_cast(req_pool_indices.data_ptr()), .req_to_token = static_cast(req_to_token.data_ptr()), - .full_to_swa = static_cast(full_to_swa.data_ptr()), + .full_to_c128_state = static_cast(full_to_c128_state.data_ptr()), .ape = static_cast(ape.data_ptr()), .state = static_cast(state.data_ptr()), .kv_score_stride_b = kv_score_input.stride(0), @@ -223,7 +218,6 @@ struct OnlineC128MTPWritePrefixKernel { .ape_stride_r = ape.stride(0), .state_stride_b = state.stride(0), .layer_bs = layer_bs, - .swa_page_size = swa_page_size, .num_verify_tokens = num_verify_tokens, .state_slot_stride = state_slot_stride, }; @@ -239,11 +233,10 @@ struct OnlineC128MTPWritePrefixKernel { tvm::ffi::TensorView seq_lens, tvm::ffi::TensorView req_pool_indices, tvm::ffi::TensorView req_to_token, - tvm::ffi::TensorView full_to_swa, + tvm::ffi::TensorView full_to_c128_state, tvm::ffi::TensorView ape, tvm::ffi::TensorView state, int64_t layer_bs, - int64_t swa_page_size, int64_t num_verify_tokens, int64_t state_slot_stride) { using namespace host; @@ -257,7 +250,7 @@ struct OnlineC128MTPWritePrefixKernel { TensorMatcher({-1}).with_dtype(seq_dtype).with_device(device).verify(seq_lens); TensorMatcher({-1}).with_dtype(req_dtype).with_device(device).verify(req_pool_indices); TensorMatcher({-1, -1}).with_dtype().with_device(device).verify(req_to_token); - TensorMatcher({-1}).with_dtype().with_device(device).verify(full_to_swa); + TensorMatcher({-1}).with_dtype().with_device(device).verify(full_to_c128_state); TensorMatcher({128, kHeadDim}).with_dtype().with_device(device).verify(ape); TensorMatcher({-1, kHeadDim * 3}).with_dtype().with_device(device).verify(state); @@ -271,62 +264,22 @@ struct OnlineC128MTPWritePrefixKernel { if (seq_dtype.is_type()) { if (req_dtype.is_type()) { launch( - kv_score_input, - seq_lens, - req_pool_indices, - req_to_token, - full_to_swa, - ape, - state, - layer_bs, - swa_page_size, - num_verify_tokens, - state_slot_stride, - device.unwrap()); + kv_score_input, seq_lens, req_pool_indices, req_to_token, full_to_c128_state, ape, state, + layer_bs, num_verify_tokens, state_slot_stride, device.unwrap()); } else { launch( - kv_score_input, - seq_lens, - req_pool_indices, - req_to_token, - full_to_swa, - ape, - state, - layer_bs, - swa_page_size, - num_verify_tokens, - state_slot_stride, - device.unwrap()); + kv_score_input, seq_lens, req_pool_indices, req_to_token, full_to_c128_state, ape, state, + layer_bs, num_verify_tokens, state_slot_stride, device.unwrap()); } } else { if (req_dtype.is_type()) { launch( - kv_score_input, - seq_lens, - req_pool_indices, - req_to_token, - full_to_swa, - ape, - state, - layer_bs, - swa_page_size, - num_verify_tokens, - state_slot_stride, - device.unwrap()); + kv_score_input, seq_lens, req_pool_indices, req_to_token, full_to_c128_state, ape, state, + layer_bs, num_verify_tokens, state_slot_stride, device.unwrap()); } else { launch( - kv_score_input, - seq_lens, - req_pool_indices, - req_to_token, - full_to_swa, - ape, - state, - layer_bs, - swa_page_size, - num_verify_tokens, - state_slot_stride, - device.unwrap()); + kv_score_input, seq_lens, req_pool_indices, req_to_token, full_to_c128_state, ape, state, + layer_bs, num_verify_tokens, state_slot_stride, device.unwrap()); } } } @@ -405,11 +358,10 @@ struct OnlineC128MTPCommitPendingKernel { tvm::ffi::TensorView cur_seq_lens, tvm::ffi::TensorView cur_req_pool_indices, tvm::ffi::TensorView req_to_token, - tvm::ffi::TensorView full_to_swa, + tvm::ffi::TensorView full_to_c128_state, tvm::ffi::TensorView pending_seq_lens, tvm::ffi::TensorView state, int64_t cur_bs, - int64_t swa_page_size, int64_t num_verify_tokens, int64_t state_slot_stride, int64_t max_num_reqs, @@ -420,13 +372,12 @@ struct OnlineC128MTPCommitPendingKernel { .cur_seq_lens = static_cast(cur_seq_lens.data_ptr()), .cur_req_pool_indices = static_cast(cur_req_pool_indices.data_ptr()), .req_to_token = static_cast(req_to_token.data_ptr()), - .full_to_swa = static_cast(full_to_swa.data_ptr()), + .full_to_c128_state = static_cast(full_to_c128_state.data_ptr()), .pending_seq_lens = static_cast(pending_seq_lens.data_ptr()), .state = static_cast(state.data_ptr()), .cur_bs = cur_bs, .req_to_token_stride_b = req_to_token.stride(0), .state_stride_b = state.stride(0), - .swa_page_size = swa_page_size, .num_verify_tokens = num_verify_tokens, .state_slot_stride = state_slot_stride, .max_num_reqs = max_num_reqs, @@ -441,11 +392,10 @@ struct OnlineC128MTPCommitPendingKernel { run(tvm::ffi::TensorView cur_seq_lens, tvm::ffi::TensorView cur_req_pool_indices, tvm::ffi::TensorView req_to_token, - tvm::ffi::TensorView full_to_swa, + tvm::ffi::TensorView full_to_c128_state, tvm::ffi::TensorView pending_seq_lens, tvm::ffi::TensorView state, int64_t cur_bs, - int64_t swa_page_size, int64_t num_verify_tokens, int64_t state_slot_stride, int64_t max_num_reqs) { @@ -459,7 +409,7 @@ struct OnlineC128MTPCommitPendingKernel { TensorMatcher({-1}).with_dtype(seq_dtype).with_device(device).verify(cur_seq_lens); TensorMatcher({-1}).with_dtype(req_dtype).with_device(device).verify(cur_req_pool_indices); TensorMatcher({-1, -1}).with_dtype().with_device(device).verify(req_to_token); - TensorMatcher({-1}).with_dtype().with_device(device).verify(full_to_swa); + TensorMatcher({-1}).with_dtype().with_device(device).verify(full_to_c128_state); TensorMatcher({-1}).with_dtype().with_device(device).verify(pending_seq_lens); TensorMatcher({-1, kHeadDim * 3}).with_dtype().with_device(device).verify(state); @@ -473,62 +423,22 @@ struct OnlineC128MTPCommitPendingKernel { if (seq_dtype.is_type()) { if (req_dtype.is_type()) { launch( - cur_seq_lens, - cur_req_pool_indices, - req_to_token, - full_to_swa, - pending_seq_lens, - state, - cur_bs, - swa_page_size, - num_verify_tokens, - state_slot_stride, - max_num_reqs, - device.unwrap()); + cur_seq_lens, cur_req_pool_indices, req_to_token, full_to_c128_state, pending_seq_lens, + state, cur_bs, num_verify_tokens, state_slot_stride, max_num_reqs, device.unwrap()); } else { launch( - cur_seq_lens, - cur_req_pool_indices, - req_to_token, - full_to_swa, - pending_seq_lens, - state, - cur_bs, - swa_page_size, - num_verify_tokens, - state_slot_stride, - max_num_reqs, - device.unwrap()); + cur_seq_lens, cur_req_pool_indices, req_to_token, full_to_c128_state, pending_seq_lens, + state, cur_bs, num_verify_tokens, state_slot_stride, max_num_reqs, device.unwrap()); } } else { if (req_dtype.is_type()) { launch( - cur_seq_lens, - cur_req_pool_indices, - req_to_token, - full_to_swa, - pending_seq_lens, - state, - cur_bs, - swa_page_size, - num_verify_tokens, - state_slot_stride, - max_num_reqs, - device.unwrap()); + cur_seq_lens, cur_req_pool_indices, req_to_token, full_to_c128_state, pending_seq_lens, + state, cur_bs, num_verify_tokens, state_slot_stride, max_num_reqs, device.unwrap()); } else { launch( - cur_seq_lens, - cur_req_pool_indices, - req_to_token, - full_to_swa, - pending_seq_lens, - state, - cur_bs, - swa_page_size, - num_verify_tokens, - state_slot_stride, - max_num_reqs, - device.unwrap()); + cur_seq_lens, cur_req_pool_indices, req_to_token, full_to_c128_state, pending_seq_lens, + state, cur_bs, num_verify_tokens, state_slot_stride, max_num_reqs, device.unwrap()); } } } diff --git a/python/sglang/jit_kernel/dsv4/compress.py b/python/sglang/jit_kernel/dsv4/compress.py index ca8d2c3421ce..6d91c99f32de 100644 --- a/python/sglang/jit_kernel/dsv4/compress.py +++ b/python/sglang/jit_kernel/dsv4/compress.py @@ -125,7 +125,7 @@ def generate( compress_ratio: Literal[4, 128], req_pool_indices: torch.Tensor, req_to_token: torch.Tensor, - full_to_swa: torch.Tensor, + full_to_state: torch.Tensor, seq_lens: torch.Tensor, swa_page_size: int, ring_size: int, @@ -134,7 +134,7 @@ def generate( plan_d = module.plan_decode( req_pool_indices, req_to_token, - full_to_swa, + full_to_state, seq_lens, int(compress_ratio), int(swa_page_size), @@ -157,8 +157,7 @@ def generate_online( seq_lens: torch.Tensor, req_pool_indices: torch.Tensor, req_to_token: torch.Tensor, - full_to_swa: torch.Tensor, - swa_page_size: int, + full_to_c128_state: torch.Tensor, state_slot_offset: int = 0, ) -> CompressorDecodePlan: batch_size = int(seq_lens.shape[0]) @@ -172,9 +171,8 @@ def generate_online( seq_lens, req_pool_indices, req_to_token, - full_to_swa, + full_to_c128_state, plan_d, - swa_page_size, int(state_slot_offset), ) return CompressorDecodePlan(128, plan_d) @@ -203,7 +201,7 @@ def generate( seq_lens: torch.Tensor, extend_lens: torch.Tensor, req_to_token: torch.Tensor, - full_to_swa: torch.Tensor, + full_to_state: torch.Tensor, swa_page_size: int, ring_size: int, num_q_tokens: int, @@ -219,7 +217,7 @@ def generate( plan_c, plan_w = module.plan_prefill( req_pool_indices, req_to_token, - full_to_swa, + full_to_state, seq_lens, extend_lens, pin_buffer, @@ -274,9 +272,8 @@ def generate_online( extend_lens: torch.Tensor, req_pool_indices: torch.Tensor, req_to_token: torch.Tensor, - full_to_swa: torch.Tensor, + full_to_c128_state: torch.Tensor, num_q_tokens: int, - swa_page_size: int, use_cuda_graph: bool = False, state_slot_offset: int = 0, ) -> CompressorPrefillPlan: @@ -284,7 +281,7 @@ def generate_online( extend_lens_cpu = extend_lens.detach().to(torch.int64).cpu() rid_i64 = req_pool_indices.to(torch.int64) r2t_i32 = req_to_token.to(torch.int32) - f2s_i64 = full_to_swa.to(torch.int64) + c128_state_i64 = full_to_c128_state.to(torch.int64) pin_buffer = torch.empty( (2, num_q_tokens, 16), dtype=torch.uint8, pin_memory=True ) @@ -298,12 +295,11 @@ def generate_online( extend_lens_cpu, rid_i64, r2t_i32, - f2s_i64, + c128_state_i64, plan_c_pin, plan_w_pin, plan_c_dev, plan_w_dev, - int(swa_page_size), int(state_slot_offset), bool(use_cuda_graph), ) diff --git a/python/sglang/jit_kernel/dsv4/online_c128_mtp.py b/python/sglang/jit_kernel/dsv4/online_c128_mtp.py index d93c6f120816..62ee66ea7979 100644 --- a/python/sglang/jit_kernel/dsv4/online_c128_mtp.py +++ b/python/sglang/jit_kernel/dsv4/online_c128_mtp.py @@ -160,11 +160,10 @@ def write_prefix_states( ctx.seq_lens, ctx.req_pool_indices, self.backend.req_to_token, - token_to_kv_pool.full_to_swa_index_mapping, + token_to_kv_pool.full_to_c128_state_index_mapping, compressor.ape.reshape(128, head_dim), state_pool.kv_score_buffer.kv_score, layer_bs, - token_to_kv_pool.swa_page_size, num_verify_tokens, state_pool.online_mtp_state_slot_offset, ) @@ -197,11 +196,10 @@ def commit_pending( seq_lens, req_pool_indices, backend.req_to_token, - token_to_kv_pool.full_to_swa_index_mapping, + token_to_kv_pool.full_to_c128_state_index_mapping, pending_seq_lens, runtime.main_state, cur_bs, - token_to_kv_pool.swa_page_size, num_verify_tokens, runtime.state_slot_offset, token_to_kv_pool.max_num_reqs, diff --git a/python/sglang/jit_kernel/tests/deepseek_v4/common.py b/python/sglang/jit_kernel/tests/deepseek_v4/common.py index 7288caa44ac3..982745c67e6e 100644 --- a/python/sglang/jit_kernel/tests/deepseek_v4/common.py +++ b/python/sglang/jit_kernel/tests/deepseek_v4/common.py @@ -103,7 +103,7 @@ def make_prefill_plan( seq_lens=seq_lens_cpu, extend_lens=extend_lens_cpu, req_to_token=self.req_to_token, - full_to_swa=self.full_to_swa, + full_to_state=self.full_to_swa, swa_page_size=self.swa_page_size, ring_size=self.ring_size, num_q_tokens=num_q_tokens, @@ -114,7 +114,7 @@ def make_decode_plan(self, seq_lens_gpu: torch.Tensor) -> CompressorDecodePlan: compress_ratio=self.compress_ratio, # type: ignore req_pool_indices=self.req_pool_indices, req_to_token=self.req_to_token, - full_to_swa=self.full_to_swa, + full_to_state=self.full_to_swa, seq_lens=seq_lens_gpu, swa_page_size=self.swa_page_size, ring_size=self.ring_size, diff --git a/python/sglang/srt/layers/attention/dsv4/compressor_v2.py b/python/sglang/srt/layers/attention/dsv4/compressor_v2.py index 7a2d266170a5..5f4a3fd0f689 100644 --- a/python/sglang/srt/layers/attention/dsv4/compressor_v2.py +++ b/python/sglang/srt/layers/attention/dsv4/compressor_v2.py @@ -708,9 +708,12 @@ def create_paged_compressor_data( swa_page_size = token_to_kv_pool.swa_page_size ring_size = token_to_kv_pool.get_ring_size(compress_ratio=compress_ratio) - # NOTE: This is actually a proxy, which encounter some bug with tvm-ffi. + # NOTE: These are actually proxies, which encounter some bug with tvm-ffi. # As a workaround, we use `.detach()` to get the real tensor. - full_to_swa = token_to_kv_pool.full_to_swa_index_mapping.detach() + if compress_ratio == 128: + full_to_state = token_to_kv_pool.full_to_c128_state_index_mapping.detach() + else: + full_to_state = token_to_kv_pool.full_to_swa_index_mapping.detach() req_pool_indices_i64 = req_pool_indices.to(torch.int64) if is_prefill: @@ -731,7 +734,7 @@ def create_paged_compressor_data( seq_lens=seq_lens_planner, extend_lens=extend_lens_planner, req_to_token=req_to_token, - full_to_swa=full_to_swa, + full_to_state=full_to_state, swa_page_size=swa_page_size, ring_size=ring_size, num_q_tokens=num_q_tokens, @@ -742,7 +745,7 @@ def create_paged_compressor_data( compress_ratio=compress_ratio, req_pool_indices=req_pool_indices_i64, req_to_token=req_to_token, - full_to_swa=full_to_swa, + full_to_state=full_to_state, seq_lens=seq_lens.to(torch.int64), swa_page_size=swa_page_size, ring_size=ring_size, @@ -763,8 +766,7 @@ def _create_online_paged_compressor_data( num_q_tokens: Optional[int], online_state_slot_offset: int = 0, ) -> CompressMetadata: - swa_page_size = int(token_to_kv_pool.swa_page_size) - full_to_swa = token_to_kv_pool.full_to_swa_index_mapping.detach() + full_to_c128_state = token_to_kv_pool.full_to_c128_state_index_mapping.detach() req_pool_indices = req_pool_indices.to(torch.int64) if is_prefill: @@ -787,9 +789,8 @@ def _create_online_paged_compressor_data( extend_lens=extend_lens_planner, req_pool_indices=req_pool_indices, req_to_token=req_to_token, - full_to_swa=full_to_swa, + full_to_c128_state=full_to_c128_state, num_q_tokens=int(num_q_tokens_planner), - swa_page_size=swa_page_size, use_cuda_graph=use_prefill_cuda_graph, state_slot_offset=online_state_slot_offset, ) @@ -798,7 +799,6 @@ def _create_online_paged_compressor_data( seq_lens=seq_lens.to(torch.int64), req_pool_indices=req_pool_indices, req_to_token=req_to_token, - full_to_swa=full_to_swa, - swa_page_size=swa_page_size, + full_to_c128_state=full_to_c128_state, state_slot_offset=online_state_slot_offset, ) diff --git a/python/sglang/srt/mem_cache/allocator/swa.py b/python/sglang/srt/mem_cache/allocator/swa.py index 31745d6457ae..52cd0609f466 100644 --- a/python/sglang/srt/mem_cache/allocator/swa.py +++ b/python/sglang/srt/mem_cache/allocator/swa.py @@ -273,6 +273,7 @@ def alloc_extend_swa_tail( self.full_to_swa_index_mapping[ alloc_full_indices[:-swa_tail_len].to(torch.int64) ] = 0 + self._set_c128_state_mapping(alloc_full_indices) return alloc_full_indices def alloc_decode( @@ -303,6 +304,7 @@ def alloc_decode( ) else: self.full_to_swa_index_mapping[alloc_full_indices] = alloc_swa_indices + self._set_c128_state_mapping(alloc_full_indices) return alloc_full_indices @@ -313,6 +315,7 @@ def free(self, free_index: torch.Tensor): # NOTE: the API is not idempotent. if self.is_not_in_free_group: self.full_attn_allocator.free(free_index) + self._clear_c128_state_mapping(free_index) self.free_swa(free_index) else: self.free_group.append(free_index) @@ -337,6 +340,17 @@ def set_full_to_swa_mapping( ) else: self.full_to_swa_index_mapping[full_indices] = swa_indices + self._set_c128_state_mapping(full_indices) + + def _set_c128_state_mapping(self, full_indices: torch.Tensor) -> None: + set_c128_mapping = getattr(self._kvcache, "set_c128_state_mapping", None) + if set_c128_mapping is not None: + set_c128_mapping(full_indices) + + def _clear_c128_state_mapping(self, full_indices: torch.Tensor) -> None: + clear_c128_mapping = getattr(self._kvcache, "clear_c128_state_mapping", None) + if clear_c128_mapping is not None: + clear_c128_mapping(full_indices) def free_swa(self, free_index: torch.Tensor): if free_index.numel() == 0: @@ -375,6 +389,9 @@ def clear(self): self.full_attn_allocator.clear() # Note: the last item is -1, we don't clear it, see the comment in __init__ self.full_to_swa_index_mapping[:-1].fill_(0) + c128_mapping = getattr(self._kvcache, "full_to_c128_state_index_mapping", None) + if c128_mapping is not None: + c128_mapping[:-1].fill_(0) self.is_not_in_free_group = True self.free_group = [] diff --git a/python/sglang/srt/mem_cache/deepseek_v4_memory_pool.py b/python/sglang/srt/mem_cache/deepseek_v4_memory_pool.py index 6d13fb6be9f2..6b45c1ecc8f6 100644 --- a/python/sglang/srt/mem_cache/deepseek_v4_memory_pool.py +++ b/python/sglang/srt/mem_cache/deepseek_v4_memory_pool.py @@ -626,11 +626,64 @@ def __init__( else: self._init_paged_compress_states(enable_memory_saver) + self._should_cache_swa = envs.SGLANG_OPT_CACHE_SWA_TRANSLATION.get() + self.cached_loc = None + self.full_to_c128_state_index_mapping: Optional[torch.Tensor] = None + def get_unified_kv(self, layer_id: int) -> torch.Tensor: return self.unified_kv_pool.get_unified_kv(layer_id - self._stage_start) def register_mapping(self, full_to_swa_index_mapping: torch.Tensor): self.full_to_swa_index_mapping = full_to_swa_index_mapping + self.full_to_c128_state_index_mapping = torch.empty_like( + full_to_swa_index_mapping + ) + full_indices = torch.arange( + full_to_swa_index_mapping.shape[0] - 1, + dtype=torch.int64, + device=full_to_swa_index_mapping.device, + ) + if ONLINE_C128: + state_indices = full_indices // 128 + else: + state_indices = full_indices + full_indices = full_indices[full_indices < self.c128_state_pool_size] + state_indices = state_indices[: full_indices.numel()] + self.full_to_c128_state_index_mapping[:-1].fill_(0) + self.full_to_c128_state_index_mapping[full_indices] = state_indices + self.full_to_c128_state_index_mapping[-1] = -1 + + def set_c128_state_mapping(self, full_indices: torch.Tensor) -> None: + if self.full_to_c128_state_index_mapping is None or full_indices.numel() == 0: + return + full_indices_i64 = full_indices.to(torch.int64) + if ONLINE_C128: + state_indices = full_indices_i64 // 128 + else: + state_indices = full_indices_i64 + self.full_to_c128_state_index_mapping[full_indices_i64] = state_indices + + def clear_c128_state_mapping(self, full_indices: torch.Tensor) -> None: + if self.full_to_c128_state_index_mapping is None or full_indices.numel() == 0: + return + full_indices = full_indices[ + (full_indices >= 0) + & (full_indices < self.full_to_c128_state_index_mapping.shape[0] - 1) + ] + if full_indices.numel() == 0: + return + full_indices = full_indices.to(torch.int64) + if self.page_size == 1: + mapping_indices = full_indices + else: + pages = torch.unique(full_indices // self.page_size) + page_offsets = torch.arange( + self.page_size, dtype=full_indices.dtype, device=full_indices.device + ) + mapping_indices = ( + pages[:, None] * self.page_size + page_offsets[None, :] + ).reshape(-1) + self.full_to_c128_state_index_mapping[mapping_indices] = 0 def get_ring_size(self, compress_ratio: int) -> int: server_args = get_global_server_args() diff --git a/python/sglang/srt/model_executor/model_runner_kv_cache_mixin.py b/python/sglang/srt/model_executor/model_runner_kv_cache_mixin.py index 1717ac95ed91..4920fc13dfb7 100644 --- a/python/sglang/srt/model_executor/model_runner_kv_cache_mixin.py +++ b/python/sglang/srt/model_executor/model_runner_kv_cache_mixin.py @@ -852,7 +852,7 @@ def _init_pools(self: ModelRunner): self.token_to_kv_pool_allocator, ) assert swa_allocator.__class__ == SWATokenToKVPoolAllocator - self.token_to_kv_pool.full_to_swa_index_mapping = ( + self.token_to_kv_pool.register_mapping( swa_allocator.full_to_swa_index_mapping ) diff --git a/python/sglang/srt/model_executor/pool_configurator.py b/python/sglang/srt/model_executor/pool_configurator.py index fb1a7f0956df..538be72ac40b 100644 --- a/python/sglang/srt/model_executor/pool_configurator.py +++ b/python/sglang/srt/model_executor/pool_configurator.py @@ -450,7 +450,10 @@ def _get_bytes_per_full_token(self) -> float: c4_indexer_state_bytes = 2 * 2 * self.indexer_head_dim * c4_state_dtype_size c4_state_ratio = self.c4_ring_size / self.swa_page_size - c128_state_ratio = self.c128_ring_size / self.swa_page_size + # C128 state is indexed from full KV positions, not SWA pages. Offline + # C128 stores one raw state row per full token; online C128 stores one + # running state per 128-token chunk. + c128_state_ratio = 1 / 128 if c128_online else 1 if c128_online and envs.SGLANG_EXPERIMENTAL_ONLINE_C128_MTP.get(): c128_state_ratio *= 1 + self.online_c128_mtp_max_draft_tokens @@ -461,10 +464,7 @@ def _get_bytes_per_full_token(self) -> float: + 1 / 128 * kv_bytes * self.num_layers_ca128 + 1 / 4 * indexer_bytes * self.num_layers_ca4 + self.swa_ratio * c4_state_ratio * c4_state_bytes * self.num_layers_ca4 - + self.swa_ratio - * c128_state_ratio - * c128_state_bytes - * self.num_layers_ca128 + + c128_state_ratio * c128_state_bytes * self.num_layers_ca128 + self.swa_ratio * c4_state_ratio * c4_indexer_state_bytes @@ -474,13 +474,15 @@ def _get_bytes_per_full_token(self) -> float: def _compute_dsv4_sizes(self, full_token: int, page_size: int) -> _DSV4PoolSizes: full_token = full_token // page_size * page_size swa_tokens = int(full_token * self.swa_ratio) // page_size * page_size + c128_online = envs.SGLANG_OPT_USE_ONLINE_COMPRESS.get() + c128_state_pool_size = full_token // 128 if c128_online else full_token return _DSV4PoolSizes( full_max_total_num_tokens=full_token, swa_max_total_num_tokens=swa_tokens, c4_max_total_num_tokens=full_token // (4 * self.c4_shrink_factor), c128_max_total_num_tokens=full_token // 128, c4_state_pool_size=swa_tokens // self.swa_page_size * self.c4_ring_size, - c128_state_pool_size=swa_tokens // self.swa_page_size * self.c128_ring_size, + c128_state_pool_size=c128_state_pool_size, ) def _to_config(self, sizes: _DSV4PoolSizes) -> MemoryPoolConfig: From b3e3f42445503758acbd71a54a8b269caed49ebe Mon Sep 17 00:00:00 2001 From: zhangxiaolei Date: Thu, 18 Jun 2026 13:55:28 +0800 Subject: [PATCH 02/24] Update PR 601 C128 state mapping optimization --- python/sglang/srt/mem_cache/allocator/swa.py | 17 -------- .../srt/mem_cache/deepseek_v4_memory_pool.py | 42 +++---------------- 2 files changed, 6 insertions(+), 53 deletions(-) diff --git a/python/sglang/srt/mem_cache/allocator/swa.py b/python/sglang/srt/mem_cache/allocator/swa.py index 52cd0609f466..31745d6457ae 100644 --- a/python/sglang/srt/mem_cache/allocator/swa.py +++ b/python/sglang/srt/mem_cache/allocator/swa.py @@ -273,7 +273,6 @@ def alloc_extend_swa_tail( self.full_to_swa_index_mapping[ alloc_full_indices[:-swa_tail_len].to(torch.int64) ] = 0 - self._set_c128_state_mapping(alloc_full_indices) return alloc_full_indices def alloc_decode( @@ -304,7 +303,6 @@ def alloc_decode( ) else: self.full_to_swa_index_mapping[alloc_full_indices] = alloc_swa_indices - self._set_c128_state_mapping(alloc_full_indices) return alloc_full_indices @@ -315,7 +313,6 @@ def free(self, free_index: torch.Tensor): # NOTE: the API is not idempotent. if self.is_not_in_free_group: self.full_attn_allocator.free(free_index) - self._clear_c128_state_mapping(free_index) self.free_swa(free_index) else: self.free_group.append(free_index) @@ -340,17 +337,6 @@ def set_full_to_swa_mapping( ) else: self.full_to_swa_index_mapping[full_indices] = swa_indices - self._set_c128_state_mapping(full_indices) - - def _set_c128_state_mapping(self, full_indices: torch.Tensor) -> None: - set_c128_mapping = getattr(self._kvcache, "set_c128_state_mapping", None) - if set_c128_mapping is not None: - set_c128_mapping(full_indices) - - def _clear_c128_state_mapping(self, full_indices: torch.Tensor) -> None: - clear_c128_mapping = getattr(self._kvcache, "clear_c128_state_mapping", None) - if clear_c128_mapping is not None: - clear_c128_mapping(full_indices) def free_swa(self, free_index: torch.Tensor): if free_index.numel() == 0: @@ -389,9 +375,6 @@ def clear(self): self.full_attn_allocator.clear() # Note: the last item is -1, we don't clear it, see the comment in __init__ self.full_to_swa_index_mapping[:-1].fill_(0) - c128_mapping = getattr(self._kvcache, "full_to_c128_state_index_mapping", None) - if c128_mapping is not None: - c128_mapping[:-1].fill_(0) self.is_not_in_free_group = True self.free_group = [] diff --git a/python/sglang/srt/mem_cache/deepseek_v4_memory_pool.py b/python/sglang/srt/mem_cache/deepseek_v4_memory_pool.py index 6b45c1ecc8f6..2bc833354980 100644 --- a/python/sglang/srt/mem_cache/deepseek_v4_memory_pool.py +++ b/python/sglang/srt/mem_cache/deepseek_v4_memory_pool.py @@ -638,8 +638,12 @@ def register_mapping(self, full_to_swa_index_mapping: torch.Tensor): self.full_to_c128_state_index_mapping = torch.empty_like( full_to_swa_index_mapping ) - full_indices = torch.arange( + valid_full_size = min( full_to_swa_index_mapping.shape[0] - 1, + self.c128_state_pool_size * (128 if ONLINE_C128 else 1), + ) + full_indices = torch.arange( + valid_full_size, dtype=torch.int64, device=full_to_swa_index_mapping.device, ) @@ -647,44 +651,10 @@ def register_mapping(self, full_to_swa_index_mapping: torch.Tensor): state_indices = full_indices // 128 else: state_indices = full_indices - full_indices = full_indices[full_indices < self.c128_state_pool_size] - state_indices = state_indices[: full_indices.numel()] self.full_to_c128_state_index_mapping[:-1].fill_(0) - self.full_to_c128_state_index_mapping[full_indices] = state_indices + self.full_to_c128_state_index_mapping[:valid_full_size] = state_indices self.full_to_c128_state_index_mapping[-1] = -1 - def set_c128_state_mapping(self, full_indices: torch.Tensor) -> None: - if self.full_to_c128_state_index_mapping is None or full_indices.numel() == 0: - return - full_indices_i64 = full_indices.to(torch.int64) - if ONLINE_C128: - state_indices = full_indices_i64 // 128 - else: - state_indices = full_indices_i64 - self.full_to_c128_state_index_mapping[full_indices_i64] = state_indices - - def clear_c128_state_mapping(self, full_indices: torch.Tensor) -> None: - if self.full_to_c128_state_index_mapping is None or full_indices.numel() == 0: - return - full_indices = full_indices[ - (full_indices >= 0) - & (full_indices < self.full_to_c128_state_index_mapping.shape[0] - 1) - ] - if full_indices.numel() == 0: - return - full_indices = full_indices.to(torch.int64) - if self.page_size == 1: - mapping_indices = full_indices - else: - pages = torch.unique(full_indices // self.page_size) - page_offsets = torch.arange( - self.page_size, dtype=full_indices.dtype, device=full_indices.device - ) - mapping_indices = ( - pages[:, None] * self.page_size + page_offsets[None, :] - ).reshape(-1) - self.full_to_c128_state_index_mapping[mapping_indices] = 0 - def get_ring_size(self, compress_ratio: int) -> int: server_args = get_global_server_args() is_speculative = server_args.speculative_algorithm is not None From 2f4573f4352dd79c0a2154a0255072dd77e3cbf0 Mon Sep 17 00:00:00 2001 From: zhangxiaolei Date: Thu, 18 Jun 2026 14:28:20 +0800 Subject: [PATCH 03/24] Optimize DSV4 C128 state slot mapping --- .../csrc/deepseek_v4/c128_online_v2.cuh | 18 ++--------- .../jit_kernel/csrc/deepseek_v4/c_plan.cuh | 25 ++++++++------- .../csrc/deepseek_v4/online_c128_mtp.cuh | 32 +++++++------------ python/sglang/jit_kernel/dsv4/compress.py | 5 --- .../sglang/jit_kernel/dsv4/online_c128_mtp.py | 2 -- .../layers/attention/dsv4/compressor_v2.py | 14 +++----- .../srt/mem_cache/deepseek_v4_memory_pool.py | 20 ------------ 7 files changed, 30 insertions(+), 86 deletions(-) diff --git a/python/sglang/jit_kernel/csrc/deepseek_v4/c128_online_v2.cuh b/python/sglang/jit_kernel/csrc/deepseek_v4/c128_online_v2.cuh index d7209960b8d5..46f54f77a64c 100644 --- a/python/sglang/jit_kernel/csrc/deepseek_v4/c128_online_v2.cuh +++ b/python/sglang/jit_kernel/csrc/deepseek_v4/c128_online_v2.cuh @@ -533,7 +533,6 @@ struct OnlineDecodePlanParams { const int64_t* __restrict__ seq_lens; const int64_t* __restrict__ req_pool_indices; const int32_t* __restrict__ req_to_token; - const int64_t* __restrict__ full_to_c128_state; // (full_cache_size,) int64 int64_t stride_r2t; int32_t state_slot_offset; uint32_t batch_size; @@ -546,7 +545,7 @@ __global__ void plan_c128_online_decode_kernel(const OnlineDecodePlanParams para const auto rid = params.req_pool_indices[idx]; const int32_t chunk_start = static_cast((seq_len - 1u) / 128u * 128u); const int32_t full_loc = params.req_to_token[rid * params.stride_r2t + chunk_start]; - const int32_t slot = static_cast(params.full_to_c128_state[full_loc]) + params.state_slot_offset; + const int32_t slot = full_loc / 128 + params.state_slot_offset; params.plan_d[idx] = DecodePlan{ .seq_len = seq_len, .write_loc = slot, @@ -564,7 +563,6 @@ inline void plan_online_decode( const tvm::ffi::TensorView seq_lens, const tvm::ffi::TensorView req_pool_indices, const tvm::ffi::TensorView req_to_token, - const tvm::ffi::TensorView full_to_c128_state, const tvm::ffi::TensorView plan_d_dev_, const int32_t state_slot_offset) { auto B = SymbolicSize{"batch_size"}; @@ -584,10 +582,6 @@ inline void plan_online_decode( .with_dtype() .with_device(device_) .verify(req_to_token); - TensorMatcher({-1}) // - .with_dtype() - .with_device(device_) - .verify(full_to_c128_state); TensorMatcher({B, sizeof(DecodePlan)}) // .with_dtype() .with_device(device_) @@ -606,7 +600,6 @@ inline void plan_online_decode( .seq_lens = static_cast(seq_lens.data_ptr()), .req_pool_indices = static_cast(req_pool_indices.data_ptr()), .req_to_token = static_cast(req_to_token.data_ptr()), - .full_to_c128_state = static_cast(full_to_c128_state.data_ptr()), .stride_r2t = stride_r2t, .state_slot_offset = state_slot_offset, .batch_size = batch_size, @@ -680,7 +673,6 @@ struct OnlinePrefillStage1Params { CompressPlan* __restrict__ plan_w; const int64_t* __restrict__ req_pool_indices; // (batch_size,) const int32_t* __restrict__ req_to_token; // (num_reqs, max_tokens) - const int64_t* __restrict__ full_to_c128_state; // (full_cache_size,) int64_t stride_r2t; int32_t state_slot_offset; uint32_t num_c; @@ -701,7 +693,7 @@ __global__ void plan_c128_online_prefill_kernel(const OnlinePrefillStage1Params const int32_t position = static_cast(plan.seq_len - 1u); const int32_t chunk_start = (position / 128) * 128; const int32_t full_loc = params.req_to_token[rid * params.stride_r2t + chunk_start]; - const int32_t main_slot = static_cast(params.full_to_c128_state[full_loc]); + const int32_t main_slot = full_loc / 128; plan.read_page_0 = main_slot + params.state_slot_offset; plan.read_page_1 = main_slot; *plan_ptr = plan; @@ -714,7 +706,6 @@ inline OnlinePrefillPlan plan_online_prefill( const tvm::ffi::TensorView extend_lens, const tvm::ffi::TensorView req_pool_indices, const tvm::ffi::TensorView req_to_token, - const tvm::ffi::TensorView full_to_c128_state, const tvm::ffi::TensorView plan_c_pin, const tvm::ffi::TensorView plan_w_pin, const tvm::ffi::TensorView plan_c_dev_, @@ -741,10 +732,6 @@ inline OnlinePrefillPlan plan_online_prefill( .with_dtype() .with_device(device_) .verify(req_to_token); - TensorMatcher({-1}) // - .with_dtype() - .with_device(device_) - .verify(full_to_c128_state); TensorMatcher({N, sizeof(CompressPlan)}) // .with_dtype() .with_device(cpu) @@ -870,7 +857,6 @@ inline OnlinePrefillPlan plan_online_prefill( .plan_w = plan_w_dev_ptr, .req_pool_indices = static_cast(req_pool_indices.data_ptr()), .req_to_token = static_cast(req_to_token.data_ptr()), - .full_to_c128_state = static_cast(full_to_c128_state.data_ptr()), .stride_r2t = req_to_token.stride(0), .state_slot_offset = state_slot_offset, .num_c = num_c_padded, diff --git a/python/sglang/jit_kernel/csrc/deepseek_v4/c_plan.cuh b/python/sglang/jit_kernel/csrc/deepseek_v4/c_plan.cuh index 7336a2b0d26c..2de9e320c5ce 100644 --- a/python/sglang/jit_kernel/csrc/deepseek_v4/c_plan.cuh +++ b/python/sglang/jit_kernel/csrc/deepseek_v4/c_plan.cuh @@ -53,7 +53,7 @@ struct Prefill1Params { PlanW* plan_w; const RID_T* rid_ptr; // [batch_size] const R2T_T* r2t_ptr; // [num_reqs, stride_r2t] - const F2S_T* state_map_ptr; // full_loc -> SWA loc for c4, C128 state loc for c128 + const F2S_T* state_map_ptr; // full_loc -> SWA loc for c4; ignored for c128 int64_t stride_r2t; uint32_t num_c; uint32_t num_w; @@ -69,7 +69,7 @@ struct DecodeParams { PlanD* plan_d; const RID_T* rid_ptr; // [batch_size] const R2T_T* r2t_ptr; // [num_reqs, stride_r2t] - const F2S_T* state_map_ptr; // full_loc -> SWA loc for c4, C128 state loc for c128 + const F2S_T* state_map_ptr; // full_loc -> SWA loc for c4; ignored for c128 const IDX_T* seq_ptr; // [batch_size] int64_t stride_r2t; uint32_t batch_size; @@ -309,12 +309,12 @@ __global__ void plan_compress_prefill_kernel_1(const Prefill1Params params) { const auto position_0 = max(position_1 - params.compress_ratio, 0); const auto raw_loc_0 = mapping[position_0]; const auto raw_loc_1 = mapping[position_1]; - const auto state_loc_0 = params.state_map_ptr[raw_loc_0]; - const auto state_loc_1 = params.state_map_ptr[raw_loc_1]; if (params.compress_ratio == 128) { - plan_c.read_page_0 = state_loc_0 / 128; - plan_c.read_page_1 = state_loc_1 / 128; + plan_c.read_page_0 = raw_loc_0 / 128; + plan_c.read_page_1 = raw_loc_1 / 128; } else { + const auto state_loc_0 = params.state_map_ptr[raw_loc_0]; + const auto state_loc_1 = params.state_map_ptr[raw_loc_1]; plan_c.read_page_0 = compute_loc(state_loc_0) / params.compress_ratio; plan_c.read_page_1 = compute_loc(state_loc_1) / params.compress_ratio; } @@ -331,10 +331,9 @@ __global__ void plan_compress_prefill_kernel_1(const Prefill1Params params) { // `seq_len` (`write_loc`) may not be aligned here const auto position = static_cast(plan_w.write_loc - 1); const auto raw_loc = mapping[position]; - const auto state_loc = params.state_map_ptr[raw_loc]; plan_w.ragged_id = ragged_id; plan_w.write_loc = - params.compress_ratio == 128 ? state_loc : compute_loc(state_loc); + params.compress_ratio == 128 ? raw_loc : compute_loc(params.state_map_ptr[raw_loc]); params.plan_w[idx] = plan_w; } else if (idx < params.num_w_padded) { params.plan_w[idx] = PlanW::invalid(); @@ -356,8 +355,10 @@ __global__ void plan_compress_decode_kernel(const DecodeParams params) { const auto position_0 = max(position_1 - params.compress_ratio, 0); const auto raw_loc_0 = mapping[position_0]; const auto raw_loc_1 = mapping[position_1]; - const auto state_loc_0 = params.state_map_ptr[raw_loc_0]; - const auto state_loc_1 = params.state_map_ptr[raw_loc_1]; + const auto state_loc_0 = + params.compress_ratio == 128 ? raw_loc_0 : params.state_map_ptr[raw_loc_0]; + const auto state_loc_1 = + params.compress_ratio == 128 ? raw_loc_1 : params.state_map_ptr[raw_loc_1]; const int32_t write_loc = static_cast( params.compress_ratio == 128 ? state_loc_1 : compute_loc(state_loc_1)); const int32_t read_page_0 = static_cast( @@ -453,8 +454,8 @@ using PrefillPlan = tvm::ffi::Tuple; * @param req_pool_indices `[batch_size]` int64_t * @param req_to_token `[num_reqs, max_tokens_per_req]` int64_t * @param full_to_state `[full_cache_size]` int64_t. For c4 this maps - * full loc -> SWA loc; for c128 it maps full loc -> - * C128 state loc. + * full loc -> SWA loc; ignored for c128, whose + * state slot is derived directly from full loc. * @param seq_lens `[batch_size]` int64 * @param extend_lens `[batch_size]` int64 * @param compress_plan `[num_q_tokens, 16]` uint8 (output) diff --git a/python/sglang/jit_kernel/csrc/deepseek_v4/online_c128_mtp.cuh b/python/sglang/jit_kernel/csrc/deepseek_v4/online_c128_mtp.cuh index c28ae49f456e..2d7279f72d0f 100644 --- a/python/sglang/jit_kernel/csrc/deepseek_v4/online_c128_mtp.cuh +++ b/python/sglang/jit_kernel/csrc/deepseek_v4/online_c128_mtp.cuh @@ -23,7 +23,6 @@ struct OnlineC128MTPWritePrefixParams { const TSeq* __restrict__ seq_lens; const TReq* __restrict__ req_pool_indices; const int32_t* __restrict__ req_to_token; - const int64_t* __restrict__ full_to_c128_state; const float* __restrict__ ape; float* __restrict__ state; int64_t kv_score_stride_b; @@ -49,7 +48,6 @@ struct OnlineC128MTPCommitPendingParams { const TSeq* __restrict__ cur_seq_lens; const TReq* __restrict__ cur_req_pool_indices; const int32_t* __restrict__ req_to_token; - const int64_t* __restrict__ full_to_c128_state; const int64_t* __restrict__ pending_seq_lens; float* __restrict__ state; int64_t cur_bs; @@ -95,7 +93,7 @@ __global__ void online_c128_mtp_commit_pending_kernel(const OnlineC128MTPCommitP const int64_t chunk_start = ((final_seq - 1) / 128) * 128; const int64_t full_loc = static_cast(params.req_to_token[req * params.req_to_token_stride_b + chunk_start]); - const int64_t slot = params.full_to_c128_state[full_loc]; + const int64_t slot = full_loc / 128; const float* const src = params.state + (slot + accept * params.state_slot_stride) * params.state_stride_b; float* const dst = params.state + slot * params.state_stride_b; @@ -119,7 +117,7 @@ __global__ void online_c128_mtp_write_prefix_kernel(const OnlineC128MTPWritePref const int64_t chunk_start = ((seq_before - 1) / 128) * 128; const int64_t full_loc = static_cast(params.req_to_token[req_idx * params.req_to_token_stride_b + chunk_start]); - init_slot = params.full_to_c128_state[full_loc]; + init_slot = full_loc / 128; } const int64_t d = static_cast(threadIdx.x); @@ -173,7 +171,7 @@ __global__ void online_c128_mtp_write_prefix_kernel(const OnlineC128MTPWritePref const int64_t chunk_start = ((final_seq - 1) / 128) * 128; const int64_t full_loc = static_cast(params.req_to_token[req_idx * params.req_to_token_stride_b + chunk_start]); - const int64_t slot = params.full_to_c128_state[full_loc] + (step + 1) * params.state_slot_stride; + const int64_t slot = full_loc / 128 + (step + 1) * params.state_slot_stride; float* const out = params.state + slot * params.state_stride_b; out[d] = run_max; out[kHeadDim + d] = run_sum; @@ -196,7 +194,6 @@ struct OnlineC128MTPWritePrefixKernel { tvm::ffi::TensorView seq_lens, tvm::ffi::TensorView req_pool_indices, tvm::ffi::TensorView req_to_token, - tvm::ffi::TensorView full_to_c128_state, tvm::ffi::TensorView ape, tvm::ffi::TensorView state, int64_t layer_bs, @@ -210,7 +207,6 @@ struct OnlineC128MTPWritePrefixKernel { .seq_lens = static_cast(seq_lens.data_ptr()), .req_pool_indices = static_cast(req_pool_indices.data_ptr()), .req_to_token = static_cast(req_to_token.data_ptr()), - .full_to_c128_state = static_cast(full_to_c128_state.data_ptr()), .ape = static_cast(ape.data_ptr()), .state = static_cast(state.data_ptr()), .kv_score_stride_b = kv_score_input.stride(0), @@ -233,7 +229,6 @@ struct OnlineC128MTPWritePrefixKernel { tvm::ffi::TensorView seq_lens, tvm::ffi::TensorView req_pool_indices, tvm::ffi::TensorView req_to_token, - tvm::ffi::TensorView full_to_c128_state, tvm::ffi::TensorView ape, tvm::ffi::TensorView state, int64_t layer_bs, @@ -250,7 +245,6 @@ struct OnlineC128MTPWritePrefixKernel { TensorMatcher({-1}).with_dtype(seq_dtype).with_device(device).verify(seq_lens); TensorMatcher({-1}).with_dtype(req_dtype).with_device(device).verify(req_pool_indices); TensorMatcher({-1, -1}).with_dtype().with_device(device).verify(req_to_token); - TensorMatcher({-1}).with_dtype().with_device(device).verify(full_to_c128_state); TensorMatcher({128, kHeadDim}).with_dtype().with_device(device).verify(ape); TensorMatcher({-1, kHeadDim * 3}).with_dtype().with_device(device).verify(state); @@ -264,21 +258,21 @@ struct OnlineC128MTPWritePrefixKernel { if (seq_dtype.is_type()) { if (req_dtype.is_type()) { launch( - kv_score_input, seq_lens, req_pool_indices, req_to_token, full_to_c128_state, ape, state, + kv_score_input, seq_lens, req_pool_indices, req_to_token, ape, state, layer_bs, num_verify_tokens, state_slot_stride, device.unwrap()); } else { launch( - kv_score_input, seq_lens, req_pool_indices, req_to_token, full_to_c128_state, ape, state, + kv_score_input, seq_lens, req_pool_indices, req_to_token, ape, state, layer_bs, num_verify_tokens, state_slot_stride, device.unwrap()); } } else { if (req_dtype.is_type()) { launch( - kv_score_input, seq_lens, req_pool_indices, req_to_token, full_to_c128_state, ape, state, + kv_score_input, seq_lens, req_pool_indices, req_to_token, ape, state, layer_bs, num_verify_tokens, state_slot_stride, device.unwrap()); } else { launch( - kv_score_input, seq_lens, req_pool_indices, req_to_token, full_to_c128_state, ape, state, + kv_score_input, seq_lens, req_pool_indices, req_to_token, ape, state, layer_bs, num_verify_tokens, state_slot_stride, device.unwrap()); } } @@ -358,7 +352,6 @@ struct OnlineC128MTPCommitPendingKernel { tvm::ffi::TensorView cur_seq_lens, tvm::ffi::TensorView cur_req_pool_indices, tvm::ffi::TensorView req_to_token, - tvm::ffi::TensorView full_to_c128_state, tvm::ffi::TensorView pending_seq_lens, tvm::ffi::TensorView state, int64_t cur_bs, @@ -372,7 +365,6 @@ struct OnlineC128MTPCommitPendingKernel { .cur_seq_lens = static_cast(cur_seq_lens.data_ptr()), .cur_req_pool_indices = static_cast(cur_req_pool_indices.data_ptr()), .req_to_token = static_cast(req_to_token.data_ptr()), - .full_to_c128_state = static_cast(full_to_c128_state.data_ptr()), .pending_seq_lens = static_cast(pending_seq_lens.data_ptr()), .state = static_cast(state.data_ptr()), .cur_bs = cur_bs, @@ -392,7 +384,6 @@ struct OnlineC128MTPCommitPendingKernel { run(tvm::ffi::TensorView cur_seq_lens, tvm::ffi::TensorView cur_req_pool_indices, tvm::ffi::TensorView req_to_token, - tvm::ffi::TensorView full_to_c128_state, tvm::ffi::TensorView pending_seq_lens, tvm::ffi::TensorView state, int64_t cur_bs, @@ -409,7 +400,6 @@ struct OnlineC128MTPCommitPendingKernel { TensorMatcher({-1}).with_dtype(seq_dtype).with_device(device).verify(cur_seq_lens); TensorMatcher({-1}).with_dtype(req_dtype).with_device(device).verify(cur_req_pool_indices); TensorMatcher({-1, -1}).with_dtype().with_device(device).verify(req_to_token); - TensorMatcher({-1}).with_dtype().with_device(device).verify(full_to_c128_state); TensorMatcher({-1}).with_dtype().with_device(device).verify(pending_seq_lens); TensorMatcher({-1, kHeadDim * 3}).with_dtype().with_device(device).verify(state); @@ -423,21 +413,21 @@ struct OnlineC128MTPCommitPendingKernel { if (seq_dtype.is_type()) { if (req_dtype.is_type()) { launch( - cur_seq_lens, cur_req_pool_indices, req_to_token, full_to_c128_state, pending_seq_lens, + cur_seq_lens, cur_req_pool_indices, req_to_token, pending_seq_lens, state, cur_bs, num_verify_tokens, state_slot_stride, max_num_reqs, device.unwrap()); } else { launch( - cur_seq_lens, cur_req_pool_indices, req_to_token, full_to_c128_state, pending_seq_lens, + cur_seq_lens, cur_req_pool_indices, req_to_token, pending_seq_lens, state, cur_bs, num_verify_tokens, state_slot_stride, max_num_reqs, device.unwrap()); } } else { if (req_dtype.is_type()) { launch( - cur_seq_lens, cur_req_pool_indices, req_to_token, full_to_c128_state, pending_seq_lens, + cur_seq_lens, cur_req_pool_indices, req_to_token, pending_seq_lens, state, cur_bs, num_verify_tokens, state_slot_stride, max_num_reqs, device.unwrap()); } else { launch( - cur_seq_lens, cur_req_pool_indices, req_to_token, full_to_c128_state, pending_seq_lens, + cur_seq_lens, cur_req_pool_indices, req_to_token, pending_seq_lens, state, cur_bs, num_verify_tokens, state_slot_stride, max_num_reqs, device.unwrap()); } } diff --git a/python/sglang/jit_kernel/dsv4/compress.py b/python/sglang/jit_kernel/dsv4/compress.py index 6d91c99f32de..7c41d5f74325 100644 --- a/python/sglang/jit_kernel/dsv4/compress.py +++ b/python/sglang/jit_kernel/dsv4/compress.py @@ -157,7 +157,6 @@ def generate_online( seq_lens: torch.Tensor, req_pool_indices: torch.Tensor, req_to_token: torch.Tensor, - full_to_c128_state: torch.Tensor, state_slot_offset: int = 0, ) -> CompressorDecodePlan: batch_size = int(seq_lens.shape[0]) @@ -171,7 +170,6 @@ def generate_online( seq_lens, req_pool_indices, req_to_token, - full_to_c128_state, plan_d, int(state_slot_offset), ) @@ -272,7 +270,6 @@ def generate_online( extend_lens: torch.Tensor, req_pool_indices: torch.Tensor, req_to_token: torch.Tensor, - full_to_c128_state: torch.Tensor, num_q_tokens: int, use_cuda_graph: bool = False, state_slot_offset: int = 0, @@ -281,7 +278,6 @@ def generate_online( extend_lens_cpu = extend_lens.detach().to(torch.int64).cpu() rid_i64 = req_pool_indices.to(torch.int64) r2t_i32 = req_to_token.to(torch.int32) - c128_state_i64 = full_to_c128_state.to(torch.int64) pin_buffer = torch.empty( (2, num_q_tokens, 16), dtype=torch.uint8, pin_memory=True ) @@ -295,7 +291,6 @@ def generate_online( extend_lens_cpu, rid_i64, r2t_i32, - c128_state_i64, plan_c_pin, plan_w_pin, plan_c_dev, diff --git a/python/sglang/jit_kernel/dsv4/online_c128_mtp.py b/python/sglang/jit_kernel/dsv4/online_c128_mtp.py index 62ee66ea7979..5948c823a59a 100644 --- a/python/sglang/jit_kernel/dsv4/online_c128_mtp.py +++ b/python/sglang/jit_kernel/dsv4/online_c128_mtp.py @@ -160,7 +160,6 @@ def write_prefix_states( ctx.seq_lens, ctx.req_pool_indices, self.backend.req_to_token, - token_to_kv_pool.full_to_c128_state_index_mapping, compressor.ape.reshape(128, head_dim), state_pool.kv_score_buffer.kv_score, layer_bs, @@ -196,7 +195,6 @@ def commit_pending( seq_lens, req_pool_indices, backend.req_to_token, - token_to_kv_pool.full_to_c128_state_index_mapping, pending_seq_lens, runtime.main_state, cur_bs, diff --git a/python/sglang/srt/layers/attention/dsv4/compressor_v2.py b/python/sglang/srt/layers/attention/dsv4/compressor_v2.py index 5f4a3fd0f689..908545ff925b 100644 --- a/python/sglang/srt/layers/attention/dsv4/compressor_v2.py +++ b/python/sglang/srt/layers/attention/dsv4/compressor_v2.py @@ -708,12 +708,9 @@ def create_paged_compressor_data( swa_page_size = token_to_kv_pool.swa_page_size ring_size = token_to_kv_pool.get_ring_size(compress_ratio=compress_ratio) - # NOTE: These are actually proxies, which encounter some bug with tvm-ffi. + # NOTE: This is actually a proxy, which encounter some bug with tvm-ffi. # As a workaround, we use `.detach()` to get the real tensor. - if compress_ratio == 128: - full_to_state = token_to_kv_pool.full_to_c128_state_index_mapping.detach() - else: - full_to_state = token_to_kv_pool.full_to_swa_index_mapping.detach() + full_to_swa = token_to_kv_pool.full_to_swa_index_mapping.detach() req_pool_indices_i64 = req_pool_indices.to(torch.int64) if is_prefill: @@ -734,7 +731,7 @@ def create_paged_compressor_data( seq_lens=seq_lens_planner, extend_lens=extend_lens_planner, req_to_token=req_to_token, - full_to_state=full_to_state, + full_to_state=full_to_swa, swa_page_size=swa_page_size, ring_size=ring_size, num_q_tokens=num_q_tokens, @@ -745,7 +742,7 @@ def create_paged_compressor_data( compress_ratio=compress_ratio, req_pool_indices=req_pool_indices_i64, req_to_token=req_to_token, - full_to_state=full_to_state, + full_to_state=full_to_swa, seq_lens=seq_lens.to(torch.int64), swa_page_size=swa_page_size, ring_size=ring_size, @@ -766,7 +763,6 @@ def _create_online_paged_compressor_data( num_q_tokens: Optional[int], online_state_slot_offset: int = 0, ) -> CompressMetadata: - full_to_c128_state = token_to_kv_pool.full_to_c128_state_index_mapping.detach() req_pool_indices = req_pool_indices.to(torch.int64) if is_prefill: @@ -789,7 +785,6 @@ def _create_online_paged_compressor_data( extend_lens=extend_lens_planner, req_pool_indices=req_pool_indices, req_to_token=req_to_token, - full_to_c128_state=full_to_c128_state, num_q_tokens=int(num_q_tokens_planner), use_cuda_graph=use_prefill_cuda_graph, state_slot_offset=online_state_slot_offset, @@ -799,6 +794,5 @@ def _create_online_paged_compressor_data( seq_lens=seq_lens.to(torch.int64), req_pool_indices=req_pool_indices, req_to_token=req_to_token, - full_to_c128_state=full_to_c128_state, state_slot_offset=online_state_slot_offset, ) diff --git a/python/sglang/srt/mem_cache/deepseek_v4_memory_pool.py b/python/sglang/srt/mem_cache/deepseek_v4_memory_pool.py index 2bc833354980..e619b7f75023 100644 --- a/python/sglang/srt/mem_cache/deepseek_v4_memory_pool.py +++ b/python/sglang/srt/mem_cache/deepseek_v4_memory_pool.py @@ -628,32 +628,12 @@ def __init__( self._should_cache_swa = envs.SGLANG_OPT_CACHE_SWA_TRANSLATION.get() self.cached_loc = None - self.full_to_c128_state_index_mapping: Optional[torch.Tensor] = None def get_unified_kv(self, layer_id: int) -> torch.Tensor: return self.unified_kv_pool.get_unified_kv(layer_id - self._stage_start) def register_mapping(self, full_to_swa_index_mapping: torch.Tensor): self.full_to_swa_index_mapping = full_to_swa_index_mapping - self.full_to_c128_state_index_mapping = torch.empty_like( - full_to_swa_index_mapping - ) - valid_full_size = min( - full_to_swa_index_mapping.shape[0] - 1, - self.c128_state_pool_size * (128 if ONLINE_C128 else 1), - ) - full_indices = torch.arange( - valid_full_size, - dtype=torch.int64, - device=full_to_swa_index_mapping.device, - ) - if ONLINE_C128: - state_indices = full_indices // 128 - else: - state_indices = full_indices - self.full_to_c128_state_index_mapping[:-1].fill_(0) - self.full_to_c128_state_index_mapping[:valid_full_size] = state_indices - self.full_to_c128_state_index_mapping[-1] = -1 def get_ring_size(self, compress_ratio: int) -> int: server_args = get_global_server_args() From 5fe855532795502293666e0c3920e1c1c078b4ae Mon Sep 17 00:00:00 2001 From: zhangxiaolei Date: Thu, 18 Jun 2026 14:38:53 +0800 Subject: [PATCH 04/24] Remove unused DSV4 SWA translation cache fields --- python/sglang/srt/mem_cache/deepseek_v4_memory_pool.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/python/sglang/srt/mem_cache/deepseek_v4_memory_pool.py b/python/sglang/srt/mem_cache/deepseek_v4_memory_pool.py index e619b7f75023..6d13fb6be9f2 100644 --- a/python/sglang/srt/mem_cache/deepseek_v4_memory_pool.py +++ b/python/sglang/srt/mem_cache/deepseek_v4_memory_pool.py @@ -626,9 +626,6 @@ def __init__( else: self._init_paged_compress_states(enable_memory_saver) - self._should_cache_swa = envs.SGLANG_OPT_CACHE_SWA_TRANSLATION.get() - self.cached_loc = None - def get_unified_kv(self, layer_id: int) -> torch.Tensor: return self.unified_kv_pool.get_unified_kv(layer_id - self._stage_start) From aa17574872073fdab887d86fddda1a53c28b497d Mon Sep 17 00:00:00 2001 From: zhangxiaolei Date: Fri, 19 Jun 2026 23:41:16 +0800 Subject: [PATCH 05/24] Optimize online C128 state pool allocation using request state pool (#603) --- .../csrc/deepseek_v4/c128_online_v2.cuh | 9 +- .../jit_kernel/csrc/deepseek_v4/c_plan.cuh | 61 +++++++++----- .../csrc/deepseek_v4/online_c128_mtp.cuh | 15 +--- python/sglang/jit_kernel/dsv4/attn.py | 27 +++--- .../sglang/jit_kernel/dsv4/online_c128_mtp.py | 4 +- python/sglang/srt/disaggregation/base/conn.py | 2 + python/sglang/srt/disaggregation/decode.py | 15 ++++ .../srt/disaggregation/mooncake/conn.py | 18 ++-- python/sglang/srt/disaggregation/mori/conn.py | 12 +-- python/sglang/srt/disaggregation/nixl/conn.py | 7 +- python/sglang/srt/disaggregation/prefill.py | 16 ++++ python/sglang/srt/disaggregation/utils.py | 12 +++ .../srt/layers/attention/dsv4/compress_hip.py | 83 ++++++++++++------- .../srt/layers/attention/dsv4/compressor.py | 11 ++- .../mem_cache/deepseek_v4_compress_state.py | 7 ++ .../srt/mem_cache/deepseek_v4_memory_pool.py | 35 +++++++- .../model_runner_kv_cache_mixin.py | 1 + .../srt/model_executor/pool_configurator.py | 32 +++++-- 18 files changed, 256 insertions(+), 111 deletions(-) diff --git a/python/sglang/jit_kernel/csrc/deepseek_v4/c128_online_v2.cuh b/python/sglang/jit_kernel/csrc/deepseek_v4/c128_online_v2.cuh index 46f54f77a64c..534527ba3e35 100644 --- a/python/sglang/jit_kernel/csrc/deepseek_v4/c128_online_v2.cuh +++ b/python/sglang/jit_kernel/csrc/deepseek_v4/c128_online_v2.cuh @@ -543,9 +543,7 @@ __global__ void plan_c128_online_decode_kernel(const OnlineDecodePlanParams para if (idx >= params.batch_size) return; const auto seq_len = static_cast(params.seq_lens[idx]); const auto rid = params.req_pool_indices[idx]; - const int32_t chunk_start = static_cast((seq_len - 1u) / 128u * 128u); - const int32_t full_loc = params.req_to_token[rid * params.stride_r2t + chunk_start]; - const int32_t slot = full_loc / 128 + params.state_slot_offset; + const int32_t slot = static_cast(rid) + params.state_slot_offset; params.plan_d[idx] = DecodePlan{ .seq_len = seq_len, .write_loc = slot, @@ -690,10 +688,7 @@ __global__ void plan_c128_online_prefill_kernel(const OnlinePrefillStage1Params if (plan.is_invalid()) return; const auto batch_id = plan.read_page_0; const auto rid = params.req_pool_indices[batch_id]; - const int32_t position = static_cast(plan.seq_len - 1u); - const int32_t chunk_start = (position / 128) * 128; - const int32_t full_loc = params.req_to_token[rid * params.stride_r2t + chunk_start]; - const int32_t main_slot = full_loc / 128; + const int32_t main_slot = static_cast(rid); plan.read_page_0 = main_slot + params.state_slot_offset; plan.read_page_1 = main_slot; *plan_ptr = plan; diff --git a/python/sglang/jit_kernel/csrc/deepseek_v4/c_plan.cuh b/python/sglang/jit_kernel/csrc/deepseek_v4/c_plan.cuh index 2de9e320c5ce..d92c3f4ad0ad 100644 --- a/python/sglang/jit_kernel/csrc/deepseek_v4/c_plan.cuh +++ b/python/sglang/jit_kernel/csrc/deepseek_v4/c_plan.cuh @@ -297,6 +297,13 @@ __global__ void plan_compress_prefill_kernel_1(const Prefill1Params params) { const auto ring_offset = swa_loc % params.ring_size; return swa_page * params.ring_size + ring_offset; }; + const auto compute_c128_page = [&](int64_t rid, int32_t position) { + const auto ring_pos = position % params.ring_size; + return static_cast(rid * (params.ring_size / 128) + ring_pos / 128); + }; + const auto compute_c128_loc = [&](int64_t rid, int32_t position) { + return static_cast(rid * params.ring_size + position % params.ring_size); + }; if (!plan_c.is_invalid()) { // 1. in bound. 2. not masked if (plan_c.buffer_len > 0) { @@ -307,12 +314,12 @@ __global__ void plan_compress_prefill_kernel_1(const Prefill1Params params) { const auto position_1 = static_cast(plan_c.seq_len - 1); // only used for c4, harmless for c128 const auto position_0 = max(position_1 - params.compress_ratio, 0); - const auto raw_loc_0 = mapping[position_0]; - const auto raw_loc_1 = mapping[position_1]; if (params.compress_ratio == 128) { - plan_c.read_page_0 = raw_loc_0 / 128; - plan_c.read_page_1 = raw_loc_1 / 128; + plan_c.read_page_0 = compute_c128_page(rid, position_0); + plan_c.read_page_1 = compute_c128_page(rid, position_1); } else { + const auto raw_loc_0 = mapping[position_0]; + const auto raw_loc_1 = mapping[position_1]; const auto state_loc_0 = params.state_map_ptr[raw_loc_0]; const auto state_loc_1 = params.state_map_ptr[raw_loc_1]; plan_c.read_page_0 = compute_loc(state_loc_0) / params.compress_ratio; @@ -330,10 +337,13 @@ __global__ void plan_compress_prefill_kernel_1(const Prefill1Params params) { const auto mapping = params.r2t_ptr + rid * params.stride_r2t; // `seq_len` (`write_loc`) may not be aligned here const auto position = static_cast(plan_w.write_loc - 1); - const auto raw_loc = mapping[position]; plan_w.ragged_id = ragged_id; - plan_w.write_loc = - params.compress_ratio == 128 ? raw_loc : compute_loc(params.state_map_ptr[raw_loc]); + if (params.compress_ratio == 128) { + plan_w.write_loc = compute_c128_loc(rid, position); + } else { + const auto raw_loc = mapping[position]; + plan_w.write_loc = compute_loc(params.state_map_ptr[raw_loc]); + } params.plan_w[idx] = plan_w; } else if (idx < params.num_w_padded) { params.plan_w[idx] = PlanW::invalid(); @@ -350,21 +360,32 @@ __global__ void plan_compress_decode_kernel(const DecodeParams params) { const auto ring_offset = swa_loc % params.ring_size; return swa_page * params.ring_size + ring_offset; }; + const auto compute_c128_page = [&](int64_t rid, int32_t position) { + const auto ring_pos = position % params.ring_size; + return static_cast(rid * (params.ring_size / 128) + ring_pos / 128); + }; + const auto compute_c128_loc = [&](int64_t rid, int32_t position) { + return static_cast(rid * params.ring_size + position % params.ring_size); + }; const auto seq_len = static_cast(params.seq_ptr[idx]); const auto position_1 = static_cast(seq_len - 1); const auto position_0 = max(position_1 - params.compress_ratio, 0); - const auto raw_loc_0 = mapping[position_0]; - const auto raw_loc_1 = mapping[position_1]; - const auto state_loc_0 = - params.compress_ratio == 128 ? raw_loc_0 : params.state_map_ptr[raw_loc_0]; - const auto state_loc_1 = - params.compress_ratio == 128 ? raw_loc_1 : params.state_map_ptr[raw_loc_1]; - const int32_t write_loc = static_cast( - params.compress_ratio == 128 ? state_loc_1 : compute_loc(state_loc_1)); - const int32_t read_page_0 = static_cast( - params.compress_ratio == 128 ? state_loc_0 / 128 : compute_loc(state_loc_0) / params.compress_ratio); - const int32_t read_page_1 = static_cast( - params.compress_ratio == 128 ? state_loc_1 / 128 : write_loc / params.compress_ratio); + int32_t write_loc; + int32_t read_page_0; + int32_t read_page_1; + if (params.compress_ratio == 128) { + write_loc = compute_c128_loc(rid, position_1); + read_page_0 = compute_c128_page(rid, position_0); + read_page_1 = compute_c128_page(rid, position_1); + } else { + const auto raw_loc_0 = mapping[position_0]; + const auto raw_loc_1 = mapping[position_1]; + const auto state_loc_0 = params.state_map_ptr[raw_loc_0]; + const auto state_loc_1 = params.state_map_ptr[raw_loc_1]; + write_loc = static_cast(compute_loc(state_loc_1)); + read_page_0 = static_cast(compute_loc(state_loc_0) / params.compress_ratio); + read_page_1 = static_cast(write_loc / params.compress_ratio); + } params.plan_d[idx] = { .seq_len = static_cast(seq_len), .write_loc = write_loc, @@ -455,7 +476,7 @@ using PrefillPlan = tvm::ffi::Tuple; * @param req_to_token `[num_reqs, max_tokens_per_req]` int64_t * @param full_to_state `[full_cache_size]` int64_t. For c4 this maps * full loc -> SWA loc; ignored for c128, whose - * state slot is derived directly from full loc. + * state slot is request-scoped. * @param seq_lens `[batch_size]` int64 * @param extend_lens `[batch_size]` int64 * @param compress_plan `[num_q_tokens, 16]` uint8 (output) diff --git a/python/sglang/jit_kernel/csrc/deepseek_v4/online_c128_mtp.cuh b/python/sglang/jit_kernel/csrc/deepseek_v4/online_c128_mtp.cuh index 2d7279f72d0f..eeed87ba03ad 100644 --- a/python/sglang/jit_kernel/csrc/deepseek_v4/online_c128_mtp.cuh +++ b/python/sglang/jit_kernel/csrc/deepseek_v4/online_c128_mtp.cuh @@ -90,10 +90,7 @@ __global__ void online_c128_mtp_commit_pending_kernel(const OnlineC128MTPCommitP const int64_t final_seq = old_seq + accept; if ((final_seq & 127) == 0) return; - const int64_t chunk_start = ((final_seq - 1) / 128) * 128; - const int64_t full_loc = - static_cast(params.req_to_token[req * params.req_to_token_stride_b + chunk_start]); - const int64_t slot = full_loc / 128; + const int64_t slot = req; const float* const src = params.state + (slot + accept * params.state_slot_stride) * params.state_stride_b; float* const dst = params.state + slot * params.state_stride_b; @@ -114,10 +111,7 @@ __global__ void online_c128_mtp_write_prefix_kernel(const OnlineC128MTPWritePref int64_t init_slot = 0; if (has_partial) { - const int64_t chunk_start = ((seq_before - 1) / 128) * 128; - const int64_t full_loc = - static_cast(params.req_to_token[req_idx * params.req_to_token_stride_b + chunk_start]); - init_slot = full_loc / 128; + init_slot = req_idx; } const int64_t d = static_cast(threadIdx.x); @@ -168,10 +162,7 @@ __global__ void online_c128_mtp_write_prefix_kernel(const OnlineC128MTPWritePref const int64_t final_seq = seq_before + step + 1; if ((final_seq & 127) != 0) { - const int64_t chunk_start = ((final_seq - 1) / 128) * 128; - const int64_t full_loc = - static_cast(params.req_to_token[req_idx * params.req_to_token_stride_b + chunk_start]); - const int64_t slot = full_loc / 128 + (step + 1) * params.state_slot_stride; + const int64_t slot = req_idx + (step + 1) * params.state_slot_stride; float* const out = params.state + slot * params.state_stride_b; out[d] = run_max; out[kHeadDim + d] = run_sum; diff --git a/python/sglang/jit_kernel/dsv4/attn.py b/python/sglang/jit_kernel/dsv4/attn.py index 87a265ecf495..784711498b27 100644 --- a/python/sglang/jit_kernel/dsv4/attn.py +++ b/python/sglang/jit_kernel/dsv4/attn.py @@ -122,18 +122,21 @@ def create_paged_compress_data_kernel( else: pos = write_overlap_pos pos = tl.maximum(pos, 0) - loc = tl.load( - req_to_token_ptr - + rid.to(tl.int64) * stride_req_to_token_0 - + pos.to(tl.int64) * stride_req_to_token_1, - mask=mask, - other=0, - ).to(tl.int32) - swa_loc = tl.load(full_to_swa_index_mapping_ptr + loc, mask=mask, other=0).to( - tl.int32 - ) - swa_page = swa_loc // swa_page_size - state_loc = swa_page * ring_size + (swa_loc % ring_size) + if compress_ratio == 128: + state_loc = rid * ring_size + (pos % ring_size) + else: + loc = tl.load( + req_to_token_ptr + + rid.to(tl.int64) * stride_req_to_token_0 + + pos.to(tl.int64) * stride_req_to_token_1, + mask=mask, + other=0, + ).to(tl.int32) + swa_loc = tl.load( + full_to_swa_index_mapping_ptr + loc, mask=mask, other=0 + ).to(tl.int32) + swa_page = swa_loc // swa_page_size + state_loc = swa_page * ring_size + (swa_loc % ring_size) state_loc = state_loc // cr if i == 0: v0 = state_loc diff --git a/python/sglang/jit_kernel/dsv4/online_c128_mtp.py b/python/sglang/jit_kernel/dsv4/online_c128_mtp.py index 5948c823a59a..a1e5394700f2 100644 --- a/python/sglang/jit_kernel/dsv4/online_c128_mtp.py +++ b/python/sglang/jit_kernel/dsv4/online_c128_mtp.py @@ -80,7 +80,7 @@ def begin_verify( req_pool_indices, token_to_kv_pool.get_online_c128_mtp_pending_seq_lens(), min(seq_lens.shape[0], req_pool_indices.shape[0]), - token_to_kv_pool.max_num_reqs, + token_to_kv_pool.get_online_c128_state_num_req_slots(), ) def clear(self) -> None: @@ -200,7 +200,7 @@ def commit_pending( cur_bs, num_verify_tokens, runtime.state_slot_offset, - token_to_kv_pool.max_num_reqs, + token_to_kv_pool.get_online_c128_state_num_req_slots(), ) self.clear() diff --git a/python/sglang/srt/disaggregation/base/conn.py b/python/sglang/srt/disaggregation/base/conn.py index f1b49b2c8634..e998e16bcceb 100644 --- a/python/sglang/srt/disaggregation/base/conn.py +++ b/python/sglang/srt/disaggregation/base/conn.py @@ -21,6 +21,8 @@ class StateType(str, enum.Enum): # DeepSeek-V4 unified_kv SWA ring: addressed per-row by ring slot # (req_pool_idx * ring_stride + pos % ring_stride), needs its own component. SWA_RING = "swa_ring" + # DeepSeek-V4 online C128 request-scoped state. + C128_STATE = "c128_state" @dataclasses.dataclass diff --git a/python/sglang/srt/disaggregation/decode.py b/python/sglang/srt/disaggregation/decode.py index 1b792e0ff3ab..397798ee3216 100644 --- a/python/sglang/srt/disaggregation/decode.py +++ b/python/sglang/srt/disaggregation/decode.py @@ -1037,6 +1037,19 @@ def _swa_ring_payload(): ring_rows = state_slot * ring_stride + (positions % ring_stride) return ring_rows.astype(np.int32) + def _c128_state_payload(): + if envs.SGLANG_OPT_USE_ONLINE_COMPRESS.get(): + return np.array([int(decode_req.req.req_pool_idx)], dtype=np.int32) + if seq_len == 0 or seq_len % 128 == 0: + return np.empty((0,), dtype=np.int32) + ring_size = self.token_to_kv_pool.get_ring_size(128) + pages_per_req = ring_size // 128 + page = ( + int(decode_req.req.req_pool_idx) * pages_per_req + + ((seq_len - 1) % ring_size) // 128 + ) + return np.array([page], dtype=np.int32) + state_types = self.kv_manager.kv_args.state_types state_indices: Optional[List] = [] for st in state_types: @@ -1048,6 +1061,8 @@ def _swa_ring_payload(): state_indices.append(_dsa_payload()) elif st == StateType.SWA_RING: state_indices.append(_swa_ring_payload()) + elif st == StateType.C128_STATE: + state_indices.append(_c128_state_payload()) else: state_indices.append(None) diff --git a/python/sglang/srt/disaggregation/mooncake/conn.py b/python/sglang/srt/disaggregation/mooncake/conn.py index f9cb07da8b98..6f9b782bf3dc 100644 --- a/python/sglang/srt/disaggregation/mooncake/conn.py +++ b/python/sglang/srt/disaggregation/mooncake/conn.py @@ -996,7 +996,12 @@ def maybe_send_extra( ) or rc ) - elif st in (StateType.SWA, StateType.DSA, StateType.SWA_RING): + elif st in ( + StateType.SWA, + StateType.DSA, + StateType.SWA_RING, + StateType.C128_STATE, + ): if ( target_rank_registration_info is not None and not self.is_mla_backend @@ -1009,12 +1014,13 @@ def maybe_send_extra( src_indices = list(indices) dst_indices_local = list(dst_indices) if len(src_indices) != len(dst_indices_local): - # SWA_RING is positional: truncating silently misaligns rows - # and corrupts KV, so fail loud. Paged SWA/DSA tolerate a - # 1-page drift -> keep the lenient truncation below. - if st == StateType.SWA_RING: + # These components are position- or request-indexed: + # truncating silently misaligns rows and corrupts KV. + # Paged SWA/DSA tolerate a 1-page drift -> keep the + # lenient truncation below. + if st in (StateType.SWA_RING, StateType.C128_STATE): raise RuntimeError( - "SWA_RING state index length mismatch: " + f"{st.upper()} state index length mismatch: " f"prefill={len(src_indices)}, dst={len(dst_indices_local)}" ) logger.warning( diff --git a/python/sglang/srt/disaggregation/mori/conn.py b/python/sglang/srt/disaggregation/mori/conn.py index 7e29f408e4b9..a7063ff3dc78 100644 --- a/python/sglang/srt/disaggregation/mori/conn.py +++ b/python/sglang/srt/disaggregation/mori/conn.py @@ -1092,7 +1092,7 @@ def send_state( dst_dims, ) ) - elif st in ("swa", "dsa", "swa_ring"): + elif st in ("swa", "dsa", "swa_ring", "c128_state"): statuses.extend( self._send_swa_dsa_state( peer_info, @@ -1226,12 +1226,12 @@ def _send_swa_dsa_state( f"No overlapping state indices for state_type={state_type}" ) if src_state_indices.size != dst_state_indices.size: - # SWA_RING is positional: truncating silently misaligns rows and - # corrupts KV, so fail loud. Paged swa/dsa tolerate a 1-page drift - # -> keep truncation. - if state_type == "swa_ring": + # These components are position- or request-indexed: truncating + # silently misaligns rows and corrupts KV. Paged swa/dsa tolerate + # a 1-page drift -> keep truncation. + if state_type in ("swa_ring", "c128_state"): raise RuntimeError( - "SWA_RING state index length mismatch: " + f"{state_type.upper()} state index length mismatch: " f"src={src_state_indices.size}, dst={dst_state_indices.size}" ) logger.warning( diff --git a/python/sglang/srt/disaggregation/nixl/conn.py b/python/sglang/srt/disaggregation/nixl/conn.py index bf20ab429c78..5bea080c7e02 100644 --- a/python/sglang/srt/disaggregation/nixl/conn.py +++ b/python/sglang/srt/disaggregation/nixl/conn.py @@ -1612,7 +1612,12 @@ def maybe_send_extra( dst_gpu_id, comp_notif, ) - elif st in (StateType.SWA, StateType.DSA, StateType.SWA_RING): + elif st in ( + StateType.SWA, + StateType.DSA, + StateType.SWA_RING, + StateType.C128_STATE, + ): if not self.is_mla_backend and self.attn_tp_size != decode_tp_size: raise RuntimeError( f"PD Disaggregation does NOT support PD different TP sizes for non-MLA {st.upper()} hybrid models yet." diff --git a/python/sglang/srt/disaggregation/prefill.py b/python/sglang/srt/disaggregation/prefill.py index a57bedb3614c..05ddbb1ff511 100644 --- a/python/sglang/srt/disaggregation/prefill.py +++ b/python/sglang/srt/disaggregation/prefill.py @@ -1007,6 +1007,20 @@ def _swa_ring_payload(): ring_rows = state_slot * ring_stride + (positions % ring_stride) return ring_rows.astype(np.int32) + def _c128_state_payload(): + if envs.SGLANG_OPT_USE_ONLINE_COMPRESS.get(): + return np.array([int(req.req_pool_idx)], dtype=np.int32) + if seq_len == 0 or seq_len % 128 == 0: + return np.empty((0,), dtype=np.int32) + _pool = self.token_to_kv_pool_allocator.get_kvcache() + ring_size = _pool.get_ring_size(128) + pages_per_req = ring_size // 128 + page = ( + int(req.req_pool_idx) * pages_per_req + + ((seq_len - 1) % ring_size) // 128 + ) + return np.array([page], dtype=np.int32) + state_types = ( self.disagg_prefill_bootstrap_queue.kv_manager.kv_args.state_types ) @@ -1020,6 +1034,8 @@ def _swa_ring_payload(): state_indices.append(_dsa_payload()) elif st == StateType.SWA_RING: state_indices.append(_swa_ring_payload()) + elif st == StateType.C128_STATE: + state_indices.append(_c128_state_payload()) else: state_indices.append(None) diff --git a/python/sglang/srt/disaggregation/utils.py b/python/sglang/srt/disaggregation/utils.py index 84bce7262a85..364bd4c3d967 100644 --- a/python/sglang/srt/disaggregation/utils.py +++ b/python/sglang/srt/disaggregation/utils.py @@ -668,6 +668,18 @@ def setup_state_kv_args( ring_lens, ring_item_lens, ) + if hasattr(token_to_kv_pool, "get_c128_state_buf_infos"): + c128_ptrs, c128_lens, c128_item_lens = ( + token_to_kv_pool.get_c128_state_buf_infos() + ) + if c128_ptrs: + append_state_component( + kv_args, + StateType.C128_STATE, + c128_ptrs, + c128_lens, + c128_item_lens, + ) elif isinstance(token_to_kv_pool, HybridLinearKVPool): dim = ( token_to_kv_pool.get_state_dim_per_tensor() diff --git a/python/sglang/srt/layers/attention/dsv4/compress_hip.py b/python/sglang/srt/layers/attention/dsv4/compress_hip.py index 8c6b7df9bd03..6ae2310f4162 100644 --- a/python/sglang/srt/layers/attention/dsv4/compress_hip.py +++ b/python/sglang/srt/layers/attention/dsv4/compress_hip.py @@ -210,13 +210,18 @@ def compress_extend_paged( pre_state_indices = self.compute_state_len_indices( seq_len=prefix_lens[i], ratio=self.ratio ).to(device) - raw_loc = torch.where( - pre_state_indices < 0, - -1, - req_to_token[req_pool_indices[i], pre_state_indices], - ) - swa_loc = token_to_kv_pool.translate_loc_from_full_to_swa(raw_loc) - state_loc = state_pool.translate_from_swa_loc_to_state_loc(swa_loc) + if self.ratio == 128: + state_loc = state_pool.translate_from_req_position_to_state_loc( + req_pool_indices[i], pre_state_indices + ) + else: + raw_loc = torch.where( + pre_state_indices < 0, + -1, + req_to_token[req_pool_indices[i], pre_state_indices], + ) + swa_loc = token_to_kv_pool.translate_loc_from_full_to_swa(raw_loc) + state_loc = state_pool.translate_from_swa_loc_to_state_loc(swa_loc) pre_kv_state = state_pool.get_state_by_state_loc(state_loc) kv_and_score_buffer = KVAndScore.cat([pre_kv_state, kv_and_score], dim=0) valid_kv_len = kv_and_score_buffer.kv.size(0) @@ -227,15 +232,22 @@ def compress_extend_paged( post_state_len = post_state_indices.size(0) assert post_state_len <= valid_kv_len - post_raw_loc = torch.where( - post_state_indices < 0, - -1, - req_to_token[req_pool_indices[i], post_state_indices], - ) - post_swa_loc = token_to_kv_pool.translate_loc_from_full_to_swa(post_raw_loc) - post_state_loc = state_pool.translate_from_swa_loc_to_state_loc( - post_swa_loc - ) + if self.ratio == 128: + post_state_loc = state_pool.translate_from_req_position_to_state_loc( + req_pool_indices[i], post_state_indices + ) + else: + post_raw_loc = torch.where( + post_state_indices < 0, + -1, + req_to_token[req_pool_indices[i], post_state_indices], + ) + post_swa_loc = token_to_kv_pool.translate_loc_from_full_to_swa( + post_raw_loc + ) + post_state_loc = state_pool.translate_from_swa_loc_to_state_loc( + post_swa_loc + ) post_state_to_set = kv_and_score_buffer[valid_kv_len - post_state_len :] state_pool.set_state_by_state_loc(post_state_loc, post_state_to_set) @@ -337,10 +349,14 @@ def compress_decode_paged( seq_lens = seq_lens_2d.view(-1) req_pool_indices = req_pool_indices.repeat_interleave(draft_tokens) - raw_locs = req_to_token[req_pool_indices, seq_lens - 1] - - swa_locs = token_to_kv_pool.translate_loc_from_full_to_swa(raw_locs) - state_locs = state_pool.translate_from_swa_loc_to_state_loc(swa_locs) + if self.ratio == 128: + state_locs = state_pool.translate_from_req_position_to_state_loc( + req_pool_indices, seq_lens - 1 + ) + else: + raw_locs = req_to_token[req_pool_indices, seq_lens - 1] + swa_locs = token_to_kv_pool.translate_loc_from_full_to_swa(raw_locs) + state_locs = state_pool.translate_from_swa_loc_to_state_loc(swa_locs) state_pool.set_state_by_state_loc(state_locs, kv_and_scores) compress_bulk_len = self.ratio * self.coff @@ -348,17 +364,22 @@ def compress_decode_paged( -compress_bulk_len, 0, device=seq_lens.device ) compress_indices.clamp_(min=-1) - compress_indices_raw = torch.where( - compress_indices < 0, - -1, - req_to_token[req_pool_indices[:, None], compress_indices], - ) - compress_indices_swa = token_to_kv_pool.translate_loc_from_full_to_swa( - compress_indices_raw - ) - compress_indices_state = state_pool.translate_from_swa_loc_to_state_loc( - compress_indices_swa - ) + if self.ratio == 128: + compress_indices_state = state_pool.translate_from_req_position_to_state_loc( + req_pool_indices[:, None], compress_indices + ) + else: + compress_indices_raw = torch.where( + compress_indices < 0, + -1, + req_to_token[req_pool_indices[:, None], compress_indices], + ) + compress_indices_swa = token_to_kv_pool.translate_loc_from_full_to_swa( + compress_indices_raw + ) + compress_indices_state = state_pool.translate_from_swa_loc_to_state_loc( + compress_indices_swa + ) kv_and_score_to_compress = state_pool.get_state_by_state_loc( compress_indices_state.view(-1) ).view(-1, self.ratio, self.coff * self.head_dim) diff --git a/python/sglang/srt/layers/attention/dsv4/compressor.py b/python/sglang/srt/layers/attention/dsv4/compressor.py index f5822ddd0542..e199f17250a1 100644 --- a/python/sglang/srt/layers/attention/dsv4/compressor.py +++ b/python/sglang/srt/layers/attention/dsv4/compressor.py @@ -283,10 +283,13 @@ def clip_down(positions: torch.Tensor) -> torch.Tensor: def get_raw_loc(positions: torch.Tensor) -> torch.Tensor: positions = positions.masked_fill(positions < 0, 0) - loc = req_to_token[req_pool_indices, positions] - swa_loc = token_to_kv_pool.translate_loc_from_full_to_swa(loc) - swa_pages = swa_loc // swa_page_size - state_loc = swa_pages * ring_size + swa_loc % ring_size + if compress_ratio == 128: + state_loc = req_pool_indices * ring_size + positions % ring_size + else: + loc = req_to_token[req_pool_indices, positions] + swa_loc = token_to_kv_pool.translate_loc_from_full_to_swa(loc) + swa_pages = swa_loc // swa_page_size + state_loc = swa_pages * ring_size + swa_loc % ring_size return (state_loc // compress_ratio).to(torch.int32) is_overlap = is_overlap_compress(compress_ratio) diff --git a/python/sglang/srt/mem_cache/deepseek_v4_compress_state.py b/python/sglang/srt/mem_cache/deepseek_v4_compress_state.py index f43415a5a49d..e52124e6c6ee 100644 --- a/python/sglang/srt/mem_cache/deepseek_v4_compress_state.py +++ b/python/sglang/srt/mem_cache/deepseek_v4_compress_state.py @@ -151,6 +151,13 @@ def translate_from_swa_loc_to_state_loc( state_loc = torch.where(swa_loc < 0, -1, state_loc) return state_loc + def translate_from_req_position_to_state_loc( + self, req_pool_indices: torch.Tensor, positions: torch.Tensor + ) -> torch.Tensor: + state_loc = req_pool_indices * self.ring_size + positions % self.ring_size + state_loc = torch.where(positions < 0, -1, state_loc) + return state_loc + def get_state_by_state_loc(self, state_loc: torch.Tensor) -> KVAndScore: return self.kv_score_buffer[state_loc] diff --git a/python/sglang/srt/mem_cache/deepseek_v4_memory_pool.py b/python/sglang/srt/mem_cache/deepseek_v4_memory_pool.py index 6d13fb6be9f2..85b5a413abd8 100644 --- a/python/sglang/srt/mem_cache/deepseek_v4_memory_pool.py +++ b/python/sglang/srt/mem_cache/deepseek_v4_memory_pool.py @@ -494,15 +494,27 @@ def __init__( self.c4_logical_size = c4_logical_size self.c128_size = c128_size self.c4_state_pool_size = c4_state_pool_size + c128_ring_size = self.get_ring_size(128) + if ONLINE_C128: + # Request-scoped online C128 state is indexed by req_pool_idx. + # PD decode can allocate pre-transfer slots beyond + # max_num_reqs, so size to the actual req_to_token row count. + c128_state_pool_size = max(c128_state_pool_size, self.num_req_slots) + else: + # Offline C128 keeps a per-request raw state ring. + c128_state_pool_size = max( + c128_state_pool_size, self.num_req_slots * c128_ring_size + ) self.c128_state_pool_size = c128_state_pool_size self.c4_state_dtype = c4_state_dtype self.c128_state_dtype = c128_state_dtype self.compression_ratios = compression_ratios self.online_mtp_max_draft_tokens = online_mtp_max_draft_tokens + self.online_c128_state_num_req_slots = c128_state_pool_size self.online_c128_mtp_pending_seq_lens: Optional[torch.Tensor] = None if ONLINE_C128 and envs.SGLANG_EXPERIMENTAL_ONLINE_C128_MTP.get(): self.online_c128_mtp_pending_seq_lens = torch.empty( - max_num_reqs, dtype=torch.int64, device=device + self.online_c128_state_num_req_slots, dtype=torch.int64, device=device ) # Determine this PP stage's absolute layer range @@ -733,6 +745,8 @@ def get_state_buf_infos(self) -> Tuple[List[int], List[int], List[int]]: for pool in pools: if pool is None: continue + if pool.ratio == 128: + continue t = pool.kv_score_buffer.kv_score assert t.ndim == 2, f"expected 2D buffer, got {t.ndim}D" data_ptrs.append(t.data_ptr()) @@ -741,6 +755,22 @@ def get_state_buf_infos(self) -> Tuple[List[int], List[int], List[int]]: return data_ptrs, data_lens, item_lens + def get_c128_state_buf_infos( + self, + ) -> Tuple[List[int], List[int], List[int]]: + data_ptrs: List[int] = [] + data_lens: List[int] = [] + item_lens: List[int] = [] + for pool in self.compress_state_pools: + if pool is None or pool.ratio != 128: + continue + t = pool.kv_score_buffer.kv_score + assert t.ndim == 2, f"expected 2D buffer, got {t.ndim}D" + data_ptrs.append(t.data_ptr()) + data_lens.append(t.nbytes) + item_lens.append(t[0].nbytes if ONLINE_C128 else t[0].nbytes * 128) + return data_ptrs, data_lens, item_lens + def _init_paged_compress_states(self, enable_memory_saver: bool): c4_state_pool_size = self.c4_state_pool_size c128_state_pool_size = self.c128_state_pool_size @@ -841,6 +871,9 @@ def get_online_c128_mtp_max_draft_tokens(self) -> int: return int(pool.online_mtp_max_draft_tokens) return 0 + def get_online_c128_state_num_req_slots(self) -> int: + return self.online_c128_state_num_req_slots + def get_online_c128_mtp_pending_seq_lens(self) -> torch.Tensor: assert self.online_c128_mtp_pending_seq_lens is not None return self.online_c128_mtp_pending_seq_lens diff --git a/python/sglang/srt/model_executor/model_runner_kv_cache_mixin.py b/python/sglang/srt/model_executor/model_runner_kv_cache_mixin.py index 4920fc13dfb7..e3859d33e0a8 100644 --- a/python/sglang/srt/model_executor/model_runner_kv_cache_mixin.py +++ b/python/sglang/srt/model_executor/model_runner_kv_cache_mixin.py @@ -1001,6 +1001,7 @@ def _resolve_memory_pool_config( config.max_running_requests = self._resolve_max_num_reqs( config.max_total_num_tokens ) + config = configurator.finalize_with_max_running_requests(config) config.mem_fraction_static = self.server_args.mem_fraction_static return config diff --git a/python/sglang/srt/model_executor/pool_configurator.py b/python/sglang/srt/model_executor/pool_configurator.py index 538be72ac40b..6ffc0b5deea8 100644 --- a/python/sglang/srt/model_executor/pool_configurator.py +++ b/python/sglang/srt/model_executor/pool_configurator.py @@ -99,6 +99,11 @@ def calculate_pool_sizes_from_max_tokens( """Constraint path: recalculate pool sizes from a constrained max_tokens.""" raise NotImplementedError + def finalize_with_max_running_requests( + self, config: MemoryPoolConfig + ) -> MemoryPoolConfig: + return config + class DefaultPoolConfigurator(MemoryPoolConfigurator): """Configurator for standard models: MHA, MLA, DSA, FP4. @@ -450,12 +455,10 @@ def _get_bytes_per_full_token(self) -> float: c4_indexer_state_bytes = 2 * 2 * self.indexer_head_dim * c4_state_dtype_size c4_state_ratio = self.c4_ring_size / self.swa_page_size - # C128 state is indexed from full KV positions, not SWA pages. Offline - # C128 stores one raw state row per full token; online C128 stores one - # running state per 128-token chunk. - c128_state_ratio = 1 / 128 if c128_online else 1 - if c128_online and envs.SGLANG_EXPERIMENTAL_ONLINE_C128_MTP.get(): - c128_state_ratio *= 1 + self.online_c128_mtp_max_draft_tokens + # C128 state is request-scoped and is finalized after + # max_running_requests is known, so it should not scale with + # full-token capacity here. + c128_state_ratio = 0 c4_frac = 1 / (4 * self.c4_shrink_factor) return ( @@ -474,15 +477,13 @@ def _get_bytes_per_full_token(self) -> float: def _compute_dsv4_sizes(self, full_token: int, page_size: int) -> _DSV4PoolSizes: full_token = full_token // page_size * page_size swa_tokens = int(full_token * self.swa_ratio) // page_size * page_size - c128_online = envs.SGLANG_OPT_USE_ONLINE_COMPRESS.get() - c128_state_pool_size = full_token // 128 if c128_online else full_token return _DSV4PoolSizes( full_max_total_num_tokens=full_token, swa_max_total_num_tokens=swa_tokens, c4_max_total_num_tokens=full_token // (4 * self.c4_shrink_factor), c128_max_total_num_tokens=full_token // 128, c4_state_pool_size=swa_tokens // self.swa_page_size * self.c4_ring_size, - c128_state_pool_size=c128_state_pool_size, + c128_state_pool_size=0, ) def _to_config(self, sizes: _DSV4PoolSizes) -> MemoryPoolConfig: @@ -505,6 +506,19 @@ def _to_config(self, sizes: _DSV4PoolSizes) -> MemoryPoolConfig: c128_state_pool_size=sizes.c128_state_pool_size, ) + def finalize_with_max_running_requests( + self, config: MemoryPoolConfig + ) -> MemoryPoolConfig: + if envs.SGLANG_OPT_USE_ONLINE_COMPRESS.get(): + assert config.max_running_requests is not None + config.c128_state_pool_size = config.max_running_requests + 1 + else: + assert config.max_running_requests is not None + config.c128_state_pool_size = ( + config.max_running_requests + 1 + ) * self.c128_ring_size + return config + def calculate_pool_sizes( self, available_bytes: int, page_size: int ) -> MemoryPoolConfig: From 263bd41041d3da653d9851f405c5a5cec35b4662 Mon Sep 17 00:00:00 2001 From: zhangxiaolei Date: Sat, 20 Jun 2026 12:07:46 +0800 Subject: [PATCH 06/24] fix review comments (#604) --- .../jit_kernel/csrc/deepseek_v4/c_plan.cuh | 38 ++++---- .../csrc/deepseek_v4/online_c128_mtp.cuh | 90 ++++--------------- .../sglang/jit_kernel/dsv4/online_c128_mtp.py | 18 ++-- python/sglang/srt/environ.py | 1 + .../layers/attention/deepseek_v4_backend.py | 21 ++--- .../deepseek_v4_backend_hip_radix.py | 20 ++--- .../srt/layers/attention/dsv4/compressor.py | 21 +---- .../srt/mem_cache/deepseek_v4_memory_pool.py | 6 +- .../srt/model_executor/pool_configurator.py | 83 +++++++++++++++-- 9 files changed, 136 insertions(+), 162 deletions(-) diff --git a/python/sglang/jit_kernel/csrc/deepseek_v4/c_plan.cuh b/python/sglang/jit_kernel/csrc/deepseek_v4/c_plan.cuh index d92c3f4ad0ad..757734dcb40e 100644 --- a/python/sglang/jit_kernel/csrc/deepseek_v4/c_plan.cuh +++ b/python/sglang/jit_kernel/csrc/deepseek_v4/c_plan.cuh @@ -53,7 +53,7 @@ struct Prefill1Params { PlanW* plan_w; const RID_T* rid_ptr; // [batch_size] const R2T_T* r2t_ptr; // [num_reqs, stride_r2t] - const F2S_T* state_map_ptr; // full_loc -> SWA loc for c4; ignored for c128 + const F2S_T* f2s_ptr; // full_loc -> SWA loc for c4; ignored for c128 int64_t stride_r2t; uint32_t num_c; uint32_t num_w; @@ -69,7 +69,7 @@ struct DecodeParams { PlanD* plan_d; const RID_T* rid_ptr; // [batch_size] const R2T_T* r2t_ptr; // [num_reqs, stride_r2t] - const F2S_T* state_map_ptr; // full_loc -> SWA loc for c4; ignored for c128 + const F2S_T* f2s_ptr; // full_loc -> SWA loc for c4; ignored for c128 const IDX_T* seq_ptr; // [batch_size] int64_t stride_r2t; uint32_t batch_size; @@ -297,10 +297,6 @@ __global__ void plan_compress_prefill_kernel_1(const Prefill1Params params) { const auto ring_offset = swa_loc % params.ring_size; return swa_page * params.ring_size + ring_offset; }; - const auto compute_c128_page = [&](int64_t rid, int32_t position) { - const auto ring_pos = position % params.ring_size; - return static_cast(rid * (params.ring_size / 128) + ring_pos / 128); - }; const auto compute_c128_loc = [&](int64_t rid, int32_t position) { return static_cast(rid * params.ring_size + position % params.ring_size); }; @@ -315,13 +311,13 @@ __global__ void plan_compress_prefill_kernel_1(const Prefill1Params params) { // only used for c4, harmless for c128 const auto position_0 = max(position_1 - params.compress_ratio, 0); if (params.compress_ratio == 128) { - plan_c.read_page_0 = compute_c128_page(rid, position_0); - plan_c.read_page_1 = compute_c128_page(rid, position_1); + plan_c.read_page_0 = compute_c128_loc(rid, position_0) / 128; + plan_c.read_page_1 = compute_c128_loc(rid, position_1) / 128; } else { const auto raw_loc_0 = mapping[position_0]; const auto raw_loc_1 = mapping[position_1]; - const auto state_loc_0 = params.state_map_ptr[raw_loc_0]; - const auto state_loc_1 = params.state_map_ptr[raw_loc_1]; + const auto state_loc_0 = params.f2s_ptr[raw_loc_0]; + const auto state_loc_1 = params.f2s_ptr[raw_loc_1]; plan_c.read_page_0 = compute_loc(state_loc_0) / params.compress_ratio; plan_c.read_page_1 = compute_loc(state_loc_1) / params.compress_ratio; } @@ -342,7 +338,7 @@ __global__ void plan_compress_prefill_kernel_1(const Prefill1Params params) { plan_w.write_loc = compute_c128_loc(rid, position); } else { const auto raw_loc = mapping[position]; - plan_w.write_loc = compute_loc(params.state_map_ptr[raw_loc]); + plan_w.write_loc = compute_loc(params.f2s_ptr[raw_loc]); } params.plan_w[idx] = plan_w; } else if (idx < params.num_w_padded) { @@ -360,10 +356,6 @@ __global__ void plan_compress_decode_kernel(const DecodeParams params) { const auto ring_offset = swa_loc % params.ring_size; return swa_page * params.ring_size + ring_offset; }; - const auto compute_c128_page = [&](int64_t rid, int32_t position) { - const auto ring_pos = position % params.ring_size; - return static_cast(rid * (params.ring_size / 128) + ring_pos / 128); - }; const auto compute_c128_loc = [&](int64_t rid, int32_t position) { return static_cast(rid * params.ring_size + position % params.ring_size); }; @@ -375,13 +367,13 @@ __global__ void plan_compress_decode_kernel(const DecodeParams params) { int32_t read_page_1; if (params.compress_ratio == 128) { write_loc = compute_c128_loc(rid, position_1); - read_page_0 = compute_c128_page(rid, position_0); - read_page_1 = compute_c128_page(rid, position_1); + read_page_0 = compute_c128_loc(rid, position_0) / 128; + read_page_1 = compute_c128_loc(rid, position_1) / 128; } else { const auto raw_loc_0 = mapping[position_0]; const auto raw_loc_1 = mapping[position_1]; - const auto state_loc_0 = params.state_map_ptr[raw_loc_0]; - const auto state_loc_1 = params.state_map_ptr[raw_loc_1]; + const auto state_loc_0 = params.f2s_ptr[raw_loc_0]; + const auto state_loc_1 = params.f2s_ptr[raw_loc_1]; write_loc = static_cast(compute_loc(state_loc_1)); read_page_0 = static_cast(compute_loc(state_loc_0) / params.compress_ratio); read_page_1 = static_cast(write_loc / params.compress_ratio); @@ -533,7 +525,7 @@ inline PrefillPlan plan_compress_prefill( const auto ext_ptr = static_cast(extend_lens.data_ptr()); const auto rid_ptr = static_cast(req_pool_indices.data_ptr()); const auto r2t_ptr = static_cast(req_to_token.data_ptr()); - const auto state_map_ptr = static_cast(full_to_state.data_ptr()); + const auto f2s_ptr = static_cast(full_to_state.data_ptr()); const auto batch_size = static_cast(B.unwrap()); constexpr auto kMaxTokens = static_cast(std::numeric_limits::max()); @@ -574,7 +566,7 @@ inline PrefillPlan plan_compress_prefill( .plan_w = static_cast(W.data_ptr()), .rid_ptr = rid_ptr, .r2t_ptr = r2t_ptr, - .state_map_ptr = state_map_ptr, + .f2s_ptr = f2s_ptr, .stride_r2t = req_to_token.stride(0), .num_c = num_q_tokens, .num_w = num_q_tokens, @@ -650,7 +642,7 @@ inline PrefillPlan plan_compress_prefill( .plan_w = static_cast(W.data_ptr()), .rid_ptr = rid_ptr, .r2t_ptr = r2t_ptr, - .state_map_ptr = state_map_ptr, + .f2s_ptr = f2s_ptr, .stride_r2t = req_to_token.size(1), .num_c = counter_c, .num_w = counter_w, @@ -703,7 +695,7 @@ inline tvm::ffi::Tensor plan_compress_decode( .plan_d = static_cast(D.data_ptr()), .rid_ptr = static_cast(req_pool_indices.data_ptr()), .r2t_ptr = static_cast(req_to_token.data_ptr()), - .state_map_ptr = static_cast(full_to_state.data_ptr()), + .f2s_ptr = static_cast(full_to_state.data_ptr()), .seq_ptr = static_cast(seq_lens.data_ptr()), .stride_r2t = req_to_token.size(1), .batch_size = batch_size, diff --git a/python/sglang/jit_kernel/csrc/deepseek_v4/online_c128_mtp.cuh b/python/sglang/jit_kernel/csrc/deepseek_v4/online_c128_mtp.cuh index eeed87ba03ad..c8a935d239de 100644 --- a/python/sglang/jit_kernel/csrc/deepseek_v4/online_c128_mtp.cuh +++ b/python/sglang/jit_kernel/csrc/deepseek_v4/online_c128_mtp.cuh @@ -177,9 +177,8 @@ __global__ void online_c128_mtp_write_prefix_kernel(const OnlineC128MTPWritePref } } -template +template struct OnlineC128MTPWritePrefixKernel { - template static void launch( tvm::ffi::TensorView kv_score_input, tvm::ffi::TensorView seq_lens, @@ -227,14 +226,12 @@ struct OnlineC128MTPWritePrefixKernel { int64_t state_slot_stride) { using namespace host; - auto seq_dtype = SymbolicDType{}; - auto req_dtype = SymbolicDType{}; auto device = SymbolicDevice{}; device.set_options(); TensorMatcher({-1, kHeadDim * 2}).with_dtype().with_device(device).verify(kv_score_input); - TensorMatcher({-1}).with_dtype(seq_dtype).with_device(device).verify(seq_lens); - TensorMatcher({-1}).with_dtype(req_dtype).with_device(device).verify(req_pool_indices); + TensorMatcher({-1}).with_dtype().with_device(device).verify(seq_lens); + TensorMatcher({-1}).with_dtype().with_device(device).verify(req_pool_indices); TensorMatcher({-1, -1}).with_dtype().with_device(device).verify(req_to_token); TensorMatcher({128, kHeadDim}).with_dtype().with_device(device).verify(ape); TensorMatcher({-1, kHeadDim * 3}).with_dtype().with_device(device).verify(state); @@ -246,33 +243,14 @@ struct OnlineC128MTPWritePrefixKernel { RuntimeCheck(layer_bs <= req_pool_indices.shape()[0], "layer_bs exceeds req_pool_indices rows"); RuntimeCheck(layer_bs * num_verify_tokens <= kv_score_input.shape()[0], "kv_score_input is too small"); - if (seq_dtype.is_type()) { - if (req_dtype.is_type()) { - launch( - kv_score_input, seq_lens, req_pool_indices, req_to_token, ape, state, - layer_bs, num_verify_tokens, state_slot_stride, device.unwrap()); - } else { - launch( - kv_score_input, seq_lens, req_pool_indices, req_to_token, ape, state, - layer_bs, num_verify_tokens, state_slot_stride, device.unwrap()); - } - } else { - if (req_dtype.is_type()) { - launch( - kv_score_input, seq_lens, req_pool_indices, req_to_token, ape, state, - layer_bs, num_verify_tokens, state_slot_stride, device.unwrap()); - } else { - launch( - kv_score_input, seq_lens, req_pool_indices, req_to_token, ape, state, - layer_bs, num_verify_tokens, state_slot_stride, device.unwrap()); - } - } + launch( + kv_score_input, seq_lens, req_pool_indices, req_to_token, ape, state, + layer_bs, num_verify_tokens, state_slot_stride, device.unwrap()); } }; -template +template struct OnlineC128MTPMarkPendingKernel { - template static void launch( tvm::ffi::TensorView seq_lens, tvm::ffi::TensorView req_pool_indices, @@ -306,13 +284,11 @@ struct OnlineC128MTPMarkPendingKernel { int64_t max_num_reqs) { using namespace host; - auto seq_dtype = SymbolicDType{}; - auto req_dtype = SymbolicDType{}; auto device = SymbolicDevice{}; device.set_options(); - TensorMatcher({-1}).with_dtype(seq_dtype).with_device(device).verify(seq_lens); - TensorMatcher({-1}).with_dtype(req_dtype).with_device(device).verify(req_pool_indices); + TensorMatcher({-1}).with_dtype().with_device(device).verify(seq_lens); + TensorMatcher({-1}).with_dtype().with_device(device).verify(req_pool_indices); TensorMatcher({-1}).with_dtype().with_device(device).verify(pending_seq_lens); if (bs <= 0) return; @@ -320,25 +296,12 @@ struct OnlineC128MTPMarkPendingKernel { RuntimeCheck(bs <= req_pool_indices.shape()[0], "bs exceeds req_pool_indices rows"); RuntimeCheck(max_num_reqs <= pending_seq_lens.shape()[0], "max_num_reqs exceeds pending rows"); - if (seq_dtype.is_type()) { - if (req_dtype.is_type()) { - launch(seq_lens, req_pool_indices, pending_seq_lens, bs, max_num_reqs, device.unwrap()); - } else { - launch(seq_lens, req_pool_indices, pending_seq_lens, bs, max_num_reqs, device.unwrap()); - } - } else { - if (req_dtype.is_type()) { - launch(seq_lens, req_pool_indices, pending_seq_lens, bs, max_num_reqs, device.unwrap()); - } else { - launch(seq_lens, req_pool_indices, pending_seq_lens, bs, max_num_reqs, device.unwrap()); - } - } + launch(seq_lens, req_pool_indices, pending_seq_lens, bs, max_num_reqs, device.unwrap()); } }; -template +template struct OnlineC128MTPCommitPendingKernel { - template static void launch( tvm::ffi::TensorView cur_seq_lens, tvm::ffi::TensorView cur_req_pool_indices, @@ -383,13 +346,11 @@ struct OnlineC128MTPCommitPendingKernel { int64_t max_num_reqs) { using namespace host; - auto seq_dtype = SymbolicDType{}; - auto req_dtype = SymbolicDType{}; auto device = SymbolicDevice{}; device.set_options(); - TensorMatcher({-1}).with_dtype(seq_dtype).with_device(device).verify(cur_seq_lens); - TensorMatcher({-1}).with_dtype(req_dtype).with_device(device).verify(cur_req_pool_indices); + TensorMatcher({-1}).with_dtype().with_device(device).verify(cur_seq_lens); + TensorMatcher({-1}).with_dtype().with_device(device).verify(cur_req_pool_indices); TensorMatcher({-1, -1}).with_dtype().with_device(device).verify(req_to_token); TensorMatcher({-1}).with_dtype().with_device(device).verify(pending_seq_lens); TensorMatcher({-1, kHeadDim * 3}).with_dtype().with_device(device).verify(state); @@ -401,27 +362,10 @@ struct OnlineC128MTPCommitPendingKernel { RuntimeCheck(cur_bs <= cur_req_pool_indices.shape()[0], "cur_bs exceeds req rows"); RuntimeCheck(max_num_reqs <= pending_seq_lens.shape()[0], "max_num_reqs exceeds pending rows"); - if (seq_dtype.is_type()) { - if (req_dtype.is_type()) { - launch( - cur_seq_lens, cur_req_pool_indices, req_to_token, pending_seq_lens, - state, cur_bs, num_verify_tokens, state_slot_stride, max_num_reqs, device.unwrap()); - } else { - launch( - cur_seq_lens, cur_req_pool_indices, req_to_token, pending_seq_lens, - state, cur_bs, num_verify_tokens, state_slot_stride, max_num_reqs, device.unwrap()); - } - } else { - if (req_dtype.is_type()) { - launch( - cur_seq_lens, cur_req_pool_indices, req_to_token, pending_seq_lens, - state, cur_bs, num_verify_tokens, state_slot_stride, max_num_reqs, device.unwrap()); - } else { - launch( - cur_seq_lens, cur_req_pool_indices, req_to_token, pending_seq_lens, - state, cur_bs, num_verify_tokens, state_slot_stride, max_num_reqs, device.unwrap()); - } - } + launch( + cur_seq_lens, cur_req_pool_indices, req_to_token, pending_seq_lens, + state, cur_bs, num_verify_tokens, state_slot_stride, max_num_reqs, + device.unwrap()); } }; diff --git a/python/sglang/jit_kernel/dsv4/online_c128_mtp.py b/python/sglang/jit_kernel/dsv4/online_c128_mtp.py index a1e5394700f2..1ec9a4df344a 100644 --- a/python/sglang/jit_kernel/dsv4/online_c128_mtp.py +++ b/python/sglang/jit_kernel/dsv4/online_c128_mtp.py @@ -12,8 +12,10 @@ @cache_once -def _jit_online_c128_mtp_module(head_dim: int) -> Module: - args = make_cpp_args(head_dim) +def _jit_online_c128_mtp_module( + head_dim: int, seq_dtype: torch.dtype, req_dtype: torch.dtype +) -> Module: + args = make_cpp_args(head_dim, seq_dtype, req_dtype) return load_jit( make_name(f"online_c128_mtp_{head_dim}"), *args, @@ -75,7 +77,9 @@ def begin_verify( if head_dim is None or self._num_verify_tokens() == 0: return token_to_kv_pool = self.backend.token_to_kv_pool - _jit_online_c128_mtp_module(head_dim).mark_pending( + _jit_online_c128_mtp_module( + head_dim, seq_lens.dtype, req_pool_indices.dtype + ).mark_pending( seq_lens, req_pool_indices, token_to_kv_pool.get_online_c128_mtp_pending_seq_lens(), @@ -155,7 +159,9 @@ def write_prefix_states( if layer_bs <= 0: return - _jit_online_c128_mtp_module(head_dim).write_prefix_states( + _jit_online_c128_mtp_module( + head_dim, ctx.seq_lens.dtype, ctx.req_pool_indices.dtype + ).write_prefix_states( kv_score_input, ctx.seq_lens, ctx.req_pool_indices, @@ -191,7 +197,9 @@ def commit_pending( cur_bs = min(seq_lens.shape[0], req_pool_indices.shape[0]) for runtime in self._iter_layer_runtimes(): - _jit_online_c128_mtp_module(runtime.head_dim).commit_pending( + _jit_online_c128_mtp_module( + runtime.head_dim, seq_lens.dtype, req_pool_indices.dtype + ).commit_pending( seq_lens, req_pool_indices, backend.req_to_token, diff --git a/python/sglang/srt/environ.py b/python/sglang/srt/environ.py index eb2027bdf8bf..e685db84a1f5 100644 --- a/python/sglang/srt/environ.py +++ b/python/sglang/srt/environ.py @@ -793,6 +793,7 @@ class Envs: SGLANG_OPT_USE_ONLINE_COMPRESS = EnvBool(False) SGLANG_EXPERIMENTAL_ONLINE_C128_MTP = EnvBool(False) SGLANG_DSV4_COMPRESS_STATE_DTYPE = EnvStr("float32") + # Deprecated: DSV4 compressor V2 is always used. SGLANG_OPT_USE_COMPRESSOR_V2 = EnvBool(True) SGLANG_FP8_PAGED_MQA_LOGITS_TORCH = EnvBool(False) SGLANG_TOPK_TRANSFORM_512_TORCH = EnvBool(False) diff --git a/python/sglang/srt/layers/attention/deepseek_v4_backend.py b/python/sglang/srt/layers/attention/deepseek_v4_backend.py index 8b504153f747..e400c36175f5 100644 --- a/python/sglang/srt/layers/attention/deepseek_v4_backend.py +++ b/python/sglang/srt/layers/attention/deepseek_v4_backend.py @@ -18,24 +18,15 @@ import torch import torch.nn.functional as F -from sglang.srt.environ import envs from sglang.srt.layers.attention.base_attn_backend import AttentionBackend +from sglang.srt.layers.attention.dsv4.compressor_v2 import ( + CompressorBackendMixin, + FusedCompressMetadata, + create_paged_compressor_data, +) +from sglang.srt.environ import envs from sglang.srt.runtime_context import get_parallel -if envs.SGLANG_OPT_USE_COMPRESSOR_V2.get(): - # NOTE: should eventually be the only compressor backend - from sglang.srt.layers.attention.dsv4.compressor_v2 import ( - CompressorBackendMixin, - FusedCompressMetadata, - create_paged_compressor_data, - ) -else: - from sglang.srt.layers.attention.dsv4.compressor import ( - CompressorBackendMixin, - FusedCompressMetadata, - create_paged_compressor_data, - ) - from sglang.jit_kernel.dsv4.online_c128_mtp import OnlineC128MTPController from sglang.srt.layers.attention.dsv4.dequant_k_cache import ( dequantize_k_cache_paged, diff --git a/python/sglang/srt/layers/attention/deepseek_v4_backend_hip_radix.py b/python/sglang/srt/layers/attention/deepseek_v4_backend_hip_radix.py index a5e55570bf5e..ade75bef1fe8 100644 --- a/python/sglang/srt/layers/attention/deepseek_v4_backend_hip_radix.py +++ b/python/sglang/srt/layers/attention/deepseek_v4_backend_hip_radix.py @@ -18,23 +18,15 @@ import torch import torch.nn.functional as F -from sglang.srt.environ import envs from sglang.srt.layers.attention.base_attn_backend import AttentionBackend +from sglang.srt.layers.attention.dsv4.compressor_v2 import ( + CompressorBackendMixin, + FusedCompressMetadata, + create_paged_compressor_data, +) +from sglang.srt.environ import envs from sglang.srt.runtime_context import get_parallel -if envs.SGLANG_OPT_USE_COMPRESSOR_V2.get(): - from sglang.srt.layers.attention.dsv4.compressor_v2 import ( - CompressorBackendMixin, - FusedCompressMetadata, - create_paged_compressor_data, - ) -else: - from sglang.srt.layers.attention.dsv4.compressor import ( - CompressorBackendMixin, - FusedCompressMetadata, - create_paged_compressor_data, - ) - from sglang.srt.layers.attention.dsv4.indexer import C4IndexerBackendMixin from sglang.srt.layers.attention.dsv4.metadata import ( PagedIndexerMetadata, diff --git a/python/sglang/srt/layers/attention/dsv4/compressor.py b/python/sglang/srt/layers/attention/dsv4/compressor.py index c7510da33bd8..5c685df35f5d 100644 --- a/python/sglang/srt/layers/attention/dsv4/compressor.py +++ b/python/sglang/srt/layers/attention/dsv4/compressor.py @@ -31,15 +31,9 @@ from sglang.srt.model_executor.forward_context import get_attn_backend from sglang.srt.models.deepseek_v2 import _is_hip from sglang.srt.runtime_context import get_parallel -from sglang.srt.utils import add_prefix, get_bool_env_var, is_npu, set_weight_attrs +from sglang.srt.utils import add_prefix, is_npu, set_weight_attrs _is_npu = is_npu() -_use_aiter = get_bool_env_var("SGLANG_USE_AITER") and _is_hip -_tgemm = None -if _use_aiter: - from aiter.tuned_gemm import tgemm - - _tgemm = tgemm if TYPE_CHECKING: from sglang.srt.layers.attention.base_attn_backend import AttentionBackend @@ -426,12 +420,7 @@ def get_state_pool(self, attn_backend: AttentionBackend) -> CompressStatePool: return ret def compute_kv_score(self, x: torch.Tensor, forward_batch: ForwardBatch): - if _tgemm is not None and not envs.SGLANG_OPT_USE_COMPRESSOR_V2.get(): - # v1 compress goes through fused_compress_triton, which promotes - # bf16->fp32 internally, so skip the .float() cast. - kv_score = _tgemm.mm(x, self.wkv_gate.weight, otype=x.dtype) - else: - kv_score = linear_bf16_fp32(x, self.wkv_gate.weight) + kv_score = linear_bf16_fp32(x, self.wkv_gate.weight) # CUDA path: delegate to backend if dsa_use_prefill_cp(forward_batch): @@ -490,9 +479,3 @@ def forward_npu( ) return get_attn_backend().forward_compress(self, x, forward_batch) - - -if _is_hip and not envs.SGLANG_OPT_USE_COMPRESSOR_V2.get(): - from sglang.srt.layers.attention.dsv4.compress_hip import ( # noqa: F811 - CompressorHip as Compressor, - ) diff --git a/python/sglang/srt/mem_cache/deepseek_v4_memory_pool.py b/python/sglang/srt/mem_cache/deepseek_v4_memory_pool.py index 21b0def4b631..bfbe1bf03f0a 100644 --- a/python/sglang/srt/mem_cache/deepseek_v4_memory_pool.py +++ b/python/sglang/srt/mem_cache/deepseek_v4_memory_pool.py @@ -614,11 +614,7 @@ def __init__( global_page_size=page_size, ) - indexer_size = ( - self.c4_logical_size - if (not _is_hip or envs.SGLANG_OPT_USE_COMPRESSOR_V2.get()) - else c4_size - ) + indexer_size = self.c4_logical_size self.c4_indexer_kv_pool = self._make_indexer_pool( indexer_size, c4_page_size, diff --git a/python/sglang/srt/model_executor/pool_configurator.py b/python/sglang/srt/model_executor/pool_configurator.py index 6ffc0b5deea8..fc40bd5fc076 100644 --- a/python/sglang/srt/model_executor/pool_configurator.py +++ b/python/sglang/srt/model_executor/pool_configurator.py @@ -28,7 +28,7 @@ from sglang.srt.layers.dp_attention import get_attention_tp_size from sglang.srt.mem_cache.deepseek_v4_memory_pool import get_compress_state_ring_size from sglang.srt.mem_cache.memory_pool import DSATokenToKVPool -from sglang.srt.utils.common import is_float4_e2m1fn_x2 +from sglang.srt.utils.common import ceil_div, is_float4_e2m1fn_x2 @dataclass @@ -359,6 +359,7 @@ def __init__(self, mr: ModelRunner): self.qk_nope_head_dim = cfg.qk_nope_head_dim self.qk_rope_head_dim = cfg.qk_rope_head_dim self.indexer_head_dim = cfg.index_head_dim + self.context_len = mr.model_config.context_len # PP-local slice; matches DeepSeekV4TokenToKVPool's stage_ratios. self.compression_ratios = cfg.compress_ratios[mr.start_layer : mr.end_layer] if mr.pp_size > 1: @@ -373,6 +374,12 @@ def __init__(self, mr: ModelRunner): self.online_c128_mtp_max_draft_tokens = ( mr.server_args.max_speculative_num_draft_tokens or 0 ) + self.requested_max_running_requests_per_worker = ( + mr.server_args.max_running_requests // mr.dp_size + if mr.server_args.max_running_requests is not None + else None + ) + self.disaggregation_mode = mr.server_args.disaggregation_mode if mr.enable_hisparse: from sglang.srt.mem_cache.sparsity import parse_hisparse_config @@ -486,6 +493,55 @@ def _compute_dsv4_sizes(self, full_token: int, page_size: int) -> _DSV4PoolSizes c128_state_pool_size=0, ) + def _get_num_req_slots(self, max_running_requests: int) -> int: + if self.disaggregation_mode == "decode": + pre_alloc_size = envs.SGLANG_DISAGGREGATION_NUM_PRE_ALLOCATE_REQS.get() + pre_alloc_size = ( + max_running_requests * 2 + if max_running_requests <= 32 + else pre_alloc_size + ) + return max_running_requests + pre_alloc_size + 1 + return max_running_requests + 1 + + def _get_c128_state_fixed_bytes(self, max_running_requests: int) -> int: + if self.num_layers_ca128 == 0: + return 0 + + _, c128_state_dtype_size = _get_dsv4_compress_state_dtype_sizes() + attn_head_dim = self.qk_nope_head_dim + self.qk_rope_head_dim + num_req_slots = self._get_num_req_slots(max_running_requests) + + if envs.SGLANG_OPT_USE_ONLINE_COMPRESS.get(): + state_rows = num_req_slots + self.c128_ring_size + 1 + state_rows *= 1 + self.online_c128_mtp_max_draft_tokens + state_last_dim = 3 * attn_head_dim + else: + state_pool_size = num_req_slots * self.c128_ring_size + state_rows = state_pool_size + self.c128_ring_size + 1 + state_rows = ceil_div(state_rows, 128) * 128 + state_last_dim = 2 * attn_head_dim + + return ( + state_rows + * state_last_dim + * c128_state_dtype_size + * self.num_layers_ca128 + ) + + def _get_c128_state_fixed_bytes_for_token_capacity( + self, token_capacity: int + ) -> int: + if self.requested_max_running_requests_per_worker is not None: + return self._get_c128_state_fixed_bytes( + self.requested_max_running_requests_per_worker + ) + + estimated = int(token_capacity / self.context_len * 512) + estimated = max(min(estimated, 4096), 2048) + max_running_requests = min(estimated, token_capacity // 2) + return self._get_c128_state_fixed_bytes(max_running_requests) + def _to_config(self, sizes: _DSV4PoolSizes) -> MemoryPoolConfig: full = sizes.full_max_total_num_tokens swa = sizes.swa_max_total_num_tokens @@ -509,14 +565,12 @@ def _to_config(self, sizes: _DSV4PoolSizes) -> MemoryPoolConfig: def finalize_with_max_running_requests( self, config: MemoryPoolConfig ) -> MemoryPoolConfig: + assert config.max_running_requests is not None + num_req_slots = self._get_num_req_slots(config.max_running_requests) if envs.SGLANG_OPT_USE_ONLINE_COMPRESS.get(): - assert config.max_running_requests is not None - config.c128_state_pool_size = config.max_running_requests + 1 + config.c128_state_pool_size = num_req_slots else: - assert config.max_running_requests is not None - config.c128_state_pool_size = ( - config.max_running_requests + 1 - ) * self.c128_ring_size + config.c128_state_pool_size = num_req_slots * self.c128_ring_size return config def calculate_pool_sizes( @@ -526,12 +580,25 @@ def calculate_pool_sizes( page_size % 128 == 0 ), "page_size must be multiple of 128 for compressed attention" - full_token = int(available_bytes / self.bytes_per_full_token) + if self.requested_max_running_requests_per_worker is not None: + c128_state_fixed_bytes = self._get_c128_state_fixed_bytes( + self.requested_max_running_requests_per_worker + ) + else: + full_token = int(available_bytes / self.bytes_per_full_token) + c128_state_fixed_bytes = ( + self._get_c128_state_fixed_bytes_for_token_capacity(full_token) + ) + + available_bytes_for_tokens = max(available_bytes - c128_state_fixed_bytes, 0) + full_token = int(available_bytes_for_tokens / self.bytes_per_full_token) + sizes = self._compute_dsv4_sizes(full_token, page_size) logger.info( f"DSV4 memory calculation: " f"bytes_per_full_token={self.bytes_per_full_token:.2f}, " f"available_bytes={available_bytes / (1 << 30):.2f} GB, " + f"c128_state_fixed={c128_state_fixed_bytes / (1 << 30):.2f} GB, " f"full_token={sizes.full_max_total_num_tokens}" ) return self._to_config(sizes) From 699a75104ea9b15607ca15a6250c31fcde602bb7 Mon Sep 17 00:00:00 2001 From: zhangxiaolei Date: Sat, 20 Jun 2026 12:26:38 +0800 Subject: [PATCH 07/24] fix review comments and lint (#605) --- .../jit_kernel/csrc/deepseek_v4/c_plan.cuh | 4 ++-- .../csrc/deepseek_v4/online_c128_mtp.cuh | 23 +++++++++++++++---- .../srt/layers/attention/dsv4/compress_hip.py | 6 +++-- .../srt/mem_cache/deepseek_v4_memory_pool.py | 1 + .../srt/model_executor/pool_configurator.py | 5 +--- 5 files changed, 27 insertions(+), 12 deletions(-) diff --git a/python/sglang/jit_kernel/csrc/deepseek_v4/c_plan.cuh b/python/sglang/jit_kernel/csrc/deepseek_v4/c_plan.cuh index 757734dcb40e..b8ddb3787977 100644 --- a/python/sglang/jit_kernel/csrc/deepseek_v4/c_plan.cuh +++ b/python/sglang/jit_kernel/csrc/deepseek_v4/c_plan.cuh @@ -53,7 +53,7 @@ struct Prefill1Params { PlanW* plan_w; const RID_T* rid_ptr; // [batch_size] const R2T_T* r2t_ptr; // [num_reqs, stride_r2t] - const F2S_T* f2s_ptr; // full_loc -> SWA loc for c4; ignored for c128 + const F2S_T* f2s_ptr; // [num_full_slots], full_loc -> swa_loc int64_t stride_r2t; uint32_t num_c; uint32_t num_w; @@ -69,7 +69,7 @@ struct DecodeParams { PlanD* plan_d; const RID_T* rid_ptr; // [batch_size] const R2T_T* r2t_ptr; // [num_reqs, stride_r2t] - const F2S_T* f2s_ptr; // full_loc -> SWA loc for c4; ignored for c128 + const F2S_T* f2s_ptr; // [num_full_slots], full_loc -> swa_loc const IDX_T* seq_ptr; // [batch_size] int64_t stride_r2t; uint32_t batch_size; diff --git a/python/sglang/jit_kernel/csrc/deepseek_v4/online_c128_mtp.cuh b/python/sglang/jit_kernel/csrc/deepseek_v4/online_c128_mtp.cuh index c8a935d239de..72eeea609eb3 100644 --- a/python/sglang/jit_kernel/csrc/deepseek_v4/online_c128_mtp.cuh +++ b/python/sglang/jit_kernel/csrc/deepseek_v4/online_c128_mtp.cuh @@ -244,8 +244,16 @@ struct OnlineC128MTPWritePrefixKernel { RuntimeCheck(layer_bs * num_verify_tokens <= kv_score_input.shape()[0], "kv_score_input is too small"); launch( - kv_score_input, seq_lens, req_pool_indices, req_to_token, ape, state, - layer_bs, num_verify_tokens, state_slot_stride, device.unwrap()); + kv_score_input, + seq_lens, + req_pool_indices, + req_to_token, + ape, + state, + layer_bs, + num_verify_tokens, + state_slot_stride, + device.unwrap()); } }; @@ -363,8 +371,15 @@ struct OnlineC128MTPCommitPendingKernel { RuntimeCheck(max_num_reqs <= pending_seq_lens.shape()[0], "max_num_reqs exceeds pending rows"); launch( - cur_seq_lens, cur_req_pool_indices, req_to_token, pending_seq_lens, - state, cur_bs, num_verify_tokens, state_slot_stride, max_num_reqs, + cur_seq_lens, + cur_req_pool_indices, + req_to_token, + pending_seq_lens, + state, + cur_bs, + num_verify_tokens, + state_slot_stride, + max_num_reqs, device.unwrap()); } }; diff --git a/python/sglang/srt/layers/attention/dsv4/compress_hip.py b/python/sglang/srt/layers/attention/dsv4/compress_hip.py index 6ae2310f4162..004d49c5bac1 100644 --- a/python/sglang/srt/layers/attention/dsv4/compress_hip.py +++ b/python/sglang/srt/layers/attention/dsv4/compress_hip.py @@ -365,8 +365,10 @@ def compress_decode_paged( ) compress_indices.clamp_(min=-1) if self.ratio == 128: - compress_indices_state = state_pool.translate_from_req_position_to_state_loc( - req_pool_indices[:, None], compress_indices + compress_indices_state = ( + state_pool.translate_from_req_position_to_state_loc( + req_pool_indices[:, None], compress_indices + ) ) else: compress_indices_raw = torch.where( diff --git a/python/sglang/srt/mem_cache/deepseek_v4_memory_pool.py b/python/sglang/srt/mem_cache/deepseek_v4_memory_pool.py index bfbe1bf03f0a..19e477f68705 100644 --- a/python/sglang/srt/mem_cache/deepseek_v4_memory_pool.py +++ b/python/sglang/srt/mem_cache/deepseek_v4_memory_pool.py @@ -764,6 +764,7 @@ def get_c128_state_buf_infos( data_lens.append(t.nbytes) item_lens.append(t[0].nbytes if ONLINE_C128 else t[0].nbytes * 128) return data_ptrs, data_lens, item_lens + def _make_kv_pool( self, *, diff --git a/python/sglang/srt/model_executor/pool_configurator.py b/python/sglang/srt/model_executor/pool_configurator.py index fc40bd5fc076..5a6112c76248 100644 --- a/python/sglang/srt/model_executor/pool_configurator.py +++ b/python/sglang/srt/model_executor/pool_configurator.py @@ -523,10 +523,7 @@ def _get_c128_state_fixed_bytes(self, max_running_requests: int) -> int: state_last_dim = 2 * attn_head_dim return ( - state_rows - * state_last_dim - * c128_state_dtype_size - * self.num_layers_ca128 + state_rows * state_last_dim * c128_state_dtype_size * self.num_layers_ca128 ) def _get_c128_state_fixed_bytes_for_token_capacity( From 3c69422cb782abc63c954f855cc808c347881a85 Mon Sep 17 00:00:00 2001 From: zhangxiaolei Date: Sat, 20 Jun 2026 12:43:31 +0800 Subject: [PATCH 08/24] Fix DSV4 backend import lint (#606) --- python/sglang/srt/layers/attention/deepseek_v4_backend.py | 7 +++---- .../srt/layers/attention/deepseek_v4_backend_hip_radix.py | 5 ++--- 2 files changed, 5 insertions(+), 7 deletions(-) diff --git a/python/sglang/srt/layers/attention/deepseek_v4_backend.py b/python/sglang/srt/layers/attention/deepseek_v4_backend.py index e400c36175f5..f9571fe774ca 100644 --- a/python/sglang/srt/layers/attention/deepseek_v4_backend.py +++ b/python/sglang/srt/layers/attention/deepseek_v4_backend.py @@ -18,16 +18,14 @@ import torch import torch.nn.functional as F +from sglang.jit_kernel.dsv4.online_c128_mtp import OnlineC128MTPController +from sglang.srt.environ import envs from sglang.srt.layers.attention.base_attn_backend import AttentionBackend from sglang.srt.layers.attention.dsv4.compressor_v2 import ( CompressorBackendMixin, FusedCompressMetadata, create_paged_compressor_data, ) -from sglang.srt.environ import envs -from sglang.srt.runtime_context import get_parallel - -from sglang.jit_kernel.dsv4.online_c128_mtp import OnlineC128MTPController from sglang.srt.layers.attention.dsv4.dequant_k_cache import ( dequantize_k_cache_paged, ) @@ -49,6 +47,7 @@ ) from sglang.srt.mem_cache.deepseek_v4_memory_pool import DeepSeekV4TokenToKVPool from sglang.srt.model_executor.forward_batch_info import ForwardBatch, ForwardMode +from sglang.srt.runtime_context import get_parallel from sglang.srt.speculative.eagle_utils import per_step_draft_out_cache_loc from sglang.srt.utils import ceil_align from sglang.srt.utils.common import is_sm120_supported diff --git a/python/sglang/srt/layers/attention/deepseek_v4_backend_hip_radix.py b/python/sglang/srt/layers/attention/deepseek_v4_backend_hip_radix.py index ade75bef1fe8..f840d7fe1aaf 100644 --- a/python/sglang/srt/layers/attention/deepseek_v4_backend_hip_radix.py +++ b/python/sglang/srt/layers/attention/deepseek_v4_backend_hip_radix.py @@ -18,15 +18,13 @@ import torch import torch.nn.functional as F +from sglang.srt.environ import envs from sglang.srt.layers.attention.base_attn_backend import AttentionBackend from sglang.srt.layers.attention.dsv4.compressor_v2 import ( CompressorBackendMixin, FusedCompressMetadata, create_paged_compressor_data, ) -from sglang.srt.environ import envs -from sglang.srt.runtime_context import get_parallel - from sglang.srt.layers.attention.dsv4.indexer import C4IndexerBackendMixin from sglang.srt.layers.attention.dsv4.metadata import ( PagedIndexerMetadata, @@ -41,6 +39,7 @@ ) from sglang.srt.mem_cache.deepseek_v4_memory_pool import DeepSeekV4TokenToKVPool from sglang.srt.model_executor.forward_batch_info import ForwardBatch, ForwardMode +from sglang.srt.runtime_context import get_parallel from sglang.srt.speculative.eagle_utils import per_step_draft_out_cache_loc from sglang.srt.utils import ceil_align From db6d3bb53087c644e342027b4c6dbee278656a17 Mon Sep 17 00:00:00 2001 From: zhangxiaolei Date: Sat, 20 Jun 2026 16:59:20 +0800 Subject: [PATCH 09/24] Fix dsv4 pd state pp slicing (#607) --- .../sglang/srt/disaggregation/common/conn.py | 63 +++++++++++-------- .../srt/disaggregation/mooncake/conn.py | 4 +- python/sglang/srt/disaggregation/nixl/conn.py | 4 +- 3 files changed, 44 insertions(+), 27 deletions(-) diff --git a/python/sglang/srt/disaggregation/common/conn.py b/python/sglang/srt/disaggregation/common/conn.py index 67115dfd2f07..8af3a2b90c95 100644 --- a/python/sglang/srt/disaggregation/common/conn.py +++ b/python/sglang/srt/disaggregation/common/conn.py @@ -23,6 +23,7 @@ KVArgs, KVPoll, KVTransferMetric, + StateType, ) from sglang.srt.disaggregation.utils import ( DisaggregationMode, @@ -521,7 +522,10 @@ def get_mha_kv_ptrs_with_pp( return src_k_ptrs, src_v_ptrs, dst_k_ptrs, dst_v_ptrs, layers_current_pp_stage def get_mla_kv_ptrs_with_pp( - self, src_kv_ptrs: List[int], dst_kv_ptrs: List[int] + self, + src_kv_ptrs: List[int], + dst_kv_ptrs: List[int], + state_type: Optional[StateType] = None, ) -> Tuple[List[int], List[int], int]: # Fast path: both sides use exactly the same PP layout if len(src_kv_ptrs) == len(dst_kv_ptrs): @@ -534,7 +538,7 @@ def get_mla_kv_ptrs_with_pp( # layer, so we locate the sub-range for this PP stage inside each # section of the dst flat list. sliced_src_kv_ptrs, sliced_dst_kv_ptrs = self._mla_slice_ptrs_for_pp( - src_kv_ptrs, dst_kv_ptrs, mla_ratios + src_kv_ptrs, dst_kv_ptrs, mla_ratios, state_type ) return ( sliced_src_kv_ptrs, @@ -554,6 +558,7 @@ def _mla_slice_ptrs_for_pp( src_kv_ptrs: List[int], dst_kv_ptrs: List[int], mla_ratios: List[int], + state_type: Optional[StateType] = None, ) -> Tuple[List[int], List[int]]: """Produce aligned (src, dst) pointer lists for compressed-MLA pools (e.g. DeepSeek V4) under PP. @@ -568,16 +573,19 @@ def _mla_slice_ptrs_for_pp( Each section is indexed by compressed-layer id within that compression bucket. - - state_data layout, length = swa_L + 2 * c4_L + c128_L: + - SWA state_data layout, length = swa_L + 2 * c4_L: [swa_layer_{0..swa_L-1}, - compress_state_{non-None, c4_L + c128_L}, - indexer_compress_state_{non-None, c4_L}] + c4_compress_state_{0..c4_L-1}, + c4_indexer_compress_state_{0..c4_L-1}] ``swa_L`` is the SWA pool's actual buffer count (``num_effective_layers``), which can be smaller than ``len(mla_ratios)`` when the HF config's ``compress_ratios`` list contains entries for layers not materialized into the SWA pool (e.g. an MTP/nextn slot at the tail). + - C128_STATE layout, length = c128_L: + [c128_compress_state_{0..c128_L-1}] + src is already PP-filtered on the prefill side. dst is the decode-side full-model list (when decode is PP=1). We slice dst to match src's PP stage. If src itself is also full-model, it is @@ -599,7 +607,18 @@ def _mla_slice_ptrs_for_pp( c128_off_s = sum(1 for r in mla_ratios[:start_layer] if r == 128) c128_off_e = sum(1 for r in mla_ratios[:end_layer] if r == 128) - if len(dst_kv_ptrs) == kv_layout_len: + if state_type == StateType.C128_STATE: + return src_kv_ptrs, list(dst_kv_ptrs[c128_off_s:c128_off_e]) + + if state_type == StateType.SWA_RING: + swa_s = min(start_layer, len(dst_kv_ptrs)) + swa_e = min(end_layer, len(dst_kv_ptrs)) + return src_kv_ptrs, list(dst_kv_ptrs[swa_s:swa_e]) + + if ( + state_type not in (StateType.SWA, StateType.SWA_RING, StateType.C128_STATE) + and len(dst_kv_ptrs) == kv_layout_len + ): sliced_dst = ( list(dst_kv_ptrs[c4_off_s:c4_off_e]) + list(dst_kv_ptrs[c4_full + c4_off_s : c4_full + c4_off_e]) @@ -607,39 +626,33 @@ def _mla_slice_ptrs_for_pp( ) return src_kv_ptrs, sliced_dst - # State-data layout. ``swa_L`` is derived from the actual dst - # length so we tolerate cases where the SWA pool has fewer - # buffers than ``len(mla_ratios)`` (e.g. nextn padding). - swa_L = len(dst_kv_ptrs) - 2 * c4_full - c128_full + # SWA state-data layout. ``swa_L`` is derived from the actual dst + # length so we tolerate cases where the SWA pool has fewer buffers + # than ``len(mla_ratios)`` (e.g. nextn padding). C128 state ships as + # a separate StateType.C128_STATE component and must not be counted + # here. + swa_L = len(dst_kv_ptrs) - 2 * c4_full if swa_L < 0 or swa_L > len(mla_ratios): raise ValueError( f"Unexpected compressed-MLA dst_kv_ptrs length " f"{len(dst_kv_ptrs)}; expected either {kv_layout_len} " - f"(kv_data) or swa_L + {2 * c4_full + c128_full} " + f"(kv_data) or swa_L + {2 * c4_full} " f"(state_data) given compression_ratios " f"(c4={c4_full}, c128={c128_full}, " f"total={len(mla_ratios)})." ) - # Guard against asking the prefill side to read past the SWA - # pool boundary. - assert end_layer <= swa_L, ( - f"prefill_end_layer ({end_layer}) exceeds dst SWA pool " - f"buffer count ({swa_L}); compression_ratios may include " - f"layers (e.g. nextn) that the SWA pool does not cover." - ) - # compress_state non-None count up to L = count(r != 0). - c_non_zero_s = sum(1 for r in mla_ratios[:start_layer] if r != 0) - c_non_zero_e = sum(1 for r in mla_ratios[:end_layer] if r != 0) + swa_s = min(start_layer, swa_L) + swa_e = min(end_layer, swa_L) compress_section_start = swa_L - indexer_section_start = swa_L + (c4_full + c128_full) + indexer_section_start = swa_L + c4_full sliced_dst = ( - list(dst_kv_ptrs[start_layer:end_layer]) + list(dst_kv_ptrs[swa_s:swa_e]) + list( dst_kv_ptrs[ compress_section_start - + c_non_zero_s : compress_section_start - + c_non_zero_e + + c4_off_s : compress_section_start + + c4_off_e ] ) + list( diff --git a/python/sglang/srt/disaggregation/mooncake/conn.py b/python/sglang/srt/disaggregation/mooncake/conn.py index 6f9b782bf3dc..ab583b90dd52 100644 --- a/python/sglang/srt/disaggregation/mooncake/conn.py +++ b/python/sglang/srt/disaggregation/mooncake/conn.py @@ -586,6 +586,7 @@ def _send_kvcache_generic( prefill_data_indices: npt.NDArray[np.int32], dst_data_indices: npt.NDArray[np.int32], executor: concurrent.futures.ThreadPoolExecutor, + state_type: Optional[StateType] = None, ) -> int: """ Generic KV cache transfer supporting both MHA and MLA architectures. @@ -601,7 +602,7 @@ def _send_kvcache_generic( # Decode pp size should be equal to prefill pp size or 1 if self.is_mla_backend: src_kv_ptrs, dst_kv_ptrs, layers_current_pp_stage = ( - self.get_mla_kv_ptrs_with_pp(src_data_ptrs, dst_data_ptrs) + self.get_mla_kv_ptrs_with_pp(src_data_ptrs, dst_data_ptrs, state_type) ) layers_params = [ ( @@ -1039,6 +1040,7 @@ def maybe_send_extra( prefill_data_indices=np.array(src_indices, dtype=np.int32), dst_data_indices=np.array(dst_indices_local, dtype=np.int32), executor=executor, + state_type=st, ) or rc ) diff --git a/python/sglang/srt/disaggregation/nixl/conn.py b/python/sglang/srt/disaggregation/nixl/conn.py index 5bea080c7e02..c9ddbd382a47 100644 --- a/python/sglang/srt/disaggregation/nixl/conn.py +++ b/python/sglang/srt/disaggregation/nixl/conn.py @@ -951,6 +951,7 @@ def _send_kvcache_generic( dst_data_indices: npt.NDArray[np.int32], dst_gpu_id: int, notif: str, + state_type: Optional[StateType] = None, ): """Generic KV cache transfer supporting both MHA and MLA architectures. Used by both send_kvcache and maybe_send_extra.""" @@ -1008,7 +1009,7 @@ def _send_kvcache_generic( # Make descs if self.is_mla_backend: src_kv_ptrs, dst_kv_ptrs, layers_current_pp_stage = ( - self.get_mla_kv_ptrs_with_pp(src_data_ptrs, dst_data_ptrs) + self.get_mla_kv_ptrs_with_pp(src_data_ptrs, dst_data_ptrs, state_type) ) layers_params = [ ( @@ -1636,6 +1637,7 @@ def maybe_send_extra( dst_data_indices=np.array(dst_indices, dtype=np.int32), dst_gpu_id=dst_gpu_id, notif=comp_notif, + state_type=st, ) else: raise RuntimeError( From 0b98a5009793b2ed1bc3232a7cedb7521e79ac94 Mon Sep 17 00:00:00 2001 From: zhangxiaolei Date: Mon, 22 Jun 2026 14:15:21 +0800 Subject: [PATCH 10/24] Align DSV4 C128 sizing extra slots (#608) --- .../sglang/srt/model_executor/pool_configurator.py | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/python/sglang/srt/model_executor/pool_configurator.py b/python/sglang/srt/model_executor/pool_configurator.py index 5fab4fa83e85..9846a6831447 100644 --- a/python/sglang/srt/model_executor/pool_configurator.py +++ b/python/sglang/srt/model_executor/pool_configurator.py @@ -30,8 +30,8 @@ from sglang.srt.mem_cache.deepseek_v4_memory_pool import get_compress_state_ring_size from sglang.srt.mem_cache.memory_pool import DSATokenToKVPool from sglang.srt.utils.common import ( - ceil_div, ceil_align, + ceil_div, is_float4_e2m1fn_x2, spec_decode_alloc_len_per_request, ) @@ -486,6 +486,9 @@ def __init__(self, mr: ModelRunner): else None ) self.disaggregation_mode = mr.server_args.disaggregation_mode + self.disaggregation_decode_extra_slots = ( + mr.server_args.disaggregation_decode_extra_slots or 0 + ) if mr.enable_hisparse: from sglang.srt.mem_cache.sparsity import parse_hisparse_config @@ -601,13 +604,7 @@ def _compute_dsv4_sizes(self, full_token: int, page_size: int) -> _DSV4PoolSizes def _get_num_req_slots(self, max_running_requests: int) -> int: if self.disaggregation_mode == "decode": - pre_alloc_size = envs.SGLANG_DISAGGREGATION_NUM_PRE_ALLOCATE_REQS.get() - pre_alloc_size = ( - max_running_requests * 2 - if max_running_requests <= 32 - else pre_alloc_size - ) - return max_running_requests + pre_alloc_size + 1 + return max_running_requests + self.disaggregation_decode_extra_slots + 1 return max_running_requests + 1 def _get_c128_state_fixed_bytes(self, max_running_requests: int) -> int: From 60ba073fa3af36b1ce92c58ed75060c7e372785c Mon Sep 17 00:00:00 2001 From: zhangxiaolei Date: Mon, 22 Jun 2026 16:51:16 +0800 Subject: [PATCH 11/24] Fix dsv4 c128 radix component (#609) --- python/sglang/srt/disaggregation/decode.py | 18 +- python/sglang/srt/disaggregation/prefill.py | 25 +-- python/sglang/srt/disaggregation/utils.py | 19 ++ python/sglang/srt/managers/schedule_policy.py | 6 +- python/sglang/srt/mem_cache/common.py | 4 + .../srt/mem_cache/deepseek_v4_memory_pool.py | 57 ++++++ .../sglang/srt/mem_cache/swa_radix_cache.py | 139 +++++++++++++- .../test_disaggregation_wire.py | 21 +++ .../mem_cache/test_dsv4_c128_radix_state.py | 177 ++++++++++++++++++ 9 files changed, 442 insertions(+), 24 deletions(-) create mode 100644 test/registered/unit/mem_cache/test_dsv4_c128_radix_state.py diff --git a/python/sglang/srt/disaggregation/decode.py b/python/sglang/srt/disaggregation/decode.py index a7f5f6537d21..15c315001293 100644 --- a/python/sglang/srt/disaggregation/decode.py +++ b/python/sglang/srt/disaggregation/decode.py @@ -50,6 +50,7 @@ ReqToMetadataIdxAllocator, TransferBackend, _is_fake_transfer, + get_dsv4_c128_state_indices, get_kv_class, is_mla_backend, poll_and_all_reduce, @@ -1041,17 +1042,14 @@ def _swa_ring_payload(): return ring_rows.astype(np.int32) def _c128_state_payload(): - if envs.SGLANG_OPT_USE_ONLINE_COMPRESS.get(): - return np.array([int(decode_req.req.req_pool_idx)], dtype=np.int32) - if seq_len == 0 or seq_len % 128 == 0: - return np.empty((0,), dtype=np.int32) - ring_size = self.token_to_kv_pool.get_ring_size(128) - pages_per_req = ring_size // 128 - page = ( - int(decode_req.req.req_pool_idx) * pages_per_req - + ((seq_len - 1) % ring_size) // 128 + online = envs.SGLANG_OPT_USE_ONLINE_COMPRESS.get() + ring_size = 1 if online else self.token_to_kv_pool.get_ring_size(128) + return get_dsv4_c128_state_indices( + int(decode_req.req.req_pool_idx), + seq_len, + online=online, + ring_size=ring_size, ) - return np.array([page], dtype=np.int32) state_types = self.kv_manager.kv_args.state_types state_indices: Optional[List] = [] diff --git a/python/sglang/srt/disaggregation/prefill.py b/python/sglang/srt/disaggregation/prefill.py index 935d241ccd76..8814603fff01 100644 --- a/python/sglang/srt/disaggregation/prefill.py +++ b/python/sglang/srt/disaggregation/prefill.py @@ -39,6 +39,7 @@ MetadataBuffers, ReqToMetadataIdxAllocator, TransferBackend, + get_dsv4_c128_state_indices, get_kv_class, is_aborted, is_mla_backend, @@ -1047,18 +1048,20 @@ def _swa_ring_payload(): return ring_rows.astype(np.int32) def _c128_state_payload(): - if envs.SGLANG_OPT_USE_ONLINE_COMPRESS.get(): - return np.array([int(req.req_pool_idx)], dtype=np.int32) - if seq_len == 0 or seq_len % 128 == 0: - return np.empty((0,), dtype=np.int32) - _pool = self.token_to_kv_pool_allocator.get_kvcache() - ring_size = _pool.get_ring_size(128) - pages_per_req = ring_size // 128 - page = ( - int(req.req_pool_idx) * pages_per_req - + ((seq_len - 1) % ring_size) // 128 + online = envs.SGLANG_OPT_USE_ONLINE_COMPRESS.get() + ring_size = ( + 1 + if online + else self.token_to_kv_pool_allocator.get_kvcache().get_ring_size( + 128 + ) + ) + return get_dsv4_c128_state_indices( + int(req.req_pool_idx), + seq_len, + online=online, + ring_size=ring_size, ) - return np.array([page], dtype=np.int32) state_types = ( self.disagg_prefill_bootstrap_queue.kv_manager.kv_args.state_types diff --git a/python/sglang/srt/disaggregation/utils.py b/python/sglang/srt/disaggregation/utils.py index 364bd4c3d967..cc98b3b5b908 100644 --- a/python/sglang/srt/disaggregation/utils.py +++ b/python/sglang/srt/disaggregation/utils.py @@ -32,6 +32,25 @@ FAKE_BOOTSTRAP_HOST = "2.2.2.2" +def get_dsv4_c128_state_indices( + req_pool_idx: int, + seq_len: int, + *, + online: bool, + ring_size: int, +) -> np.ndarray: + """Return the PD transfer row/page indices for DSV4 C128 state.""" + if online: + return np.array([int(req_pool_idx)], dtype=np.int32) + if seq_len == 0 or seq_len % 128 == 0: + return np.empty((0,), dtype=np.int32) + + assert ring_size % 128 == 0, f"C128 ring_size must be 128-aligned, got {ring_size}" + pages_per_req = ring_size // 128 + page = int(req_pool_idx) * pages_per_req + ((seq_len - 1) % ring_size) // 128 + return np.array([page], dtype=np.int32) + + class DisaggregationMode(Enum): NULL = "null" PREFILL = "prefill" diff --git a/python/sglang/srt/managers/schedule_policy.py b/python/sglang/srt/managers/schedule_policy.py index 1b34f0f54b6d..4b5bd9686ffd 100644 --- a/python/sglang/srt/managers/schedule_policy.py +++ b/python/sglang/srt/managers/schedule_policy.py @@ -182,7 +182,7 @@ def calc_priority( and get_global_server_args().disaggregation_mode != "decode" ): for r in waiting_queue: - match_prefix_for_req(self.tree_cache, r) + match_prefix_for_req(self.tree_cache, r, include_req=True) if self.policy == CacheAgnosticPolicy.FCFS: if self.enable_priority_scheduling: @@ -257,7 +257,9 @@ def _compute_prefix_matches( for r in waiting_queue: prefix_ids = r.origin_input_ids + r.output_ids extra_key = r.extra_key - match_result = match_prefix_for_req(self.tree_cache, r, prefix_ids) + match_result = match_prefix_for_req( + self.tree_cache, r, prefix_ids, include_req=True + ) # NOTE(sang): This logic is for in-batch prefix caching; # If there are more than 1 request that have small matching prefix from diff --git a/python/sglang/srt/mem_cache/common.py b/python/sglang/srt/mem_cache/common.py index 0d10b7947df5..fc65c8d83866 100644 --- a/python/sglang/srt/mem_cache/common.py +++ b/python/sglang/srt/mem_cache/common.py @@ -505,6 +505,10 @@ def alloc_for_extend( batch.req_to_token_pool, ) + restore_c128_state = getattr(batch.tree_cache, "restore_c128_state_for_reqs", None) + if restore_c128_state is not None: + restore_c128_state(batch.reqs) + # DSV4-NPU hook: write c4/c128/swa per-req tables from the stashed bundle. # No-op on non-DSV4 paths (out_cache_loc_dsv4 stays None there). if _is_npu: diff --git a/python/sglang/srt/mem_cache/deepseek_v4_memory_pool.py b/python/sglang/srt/mem_cache/deepseek_v4_memory_pool.py index 19e477f68705..d0f363e3d87d 100644 --- a/python/sglang/srt/mem_cache/deepseek_v4_memory_pool.py +++ b/python/sglang/srt/mem_cache/deepseek_v4_memory_pool.py @@ -942,6 +942,63 @@ def get_online_c128_mtp_pending_seq_lens(self) -> torch.Tensor: assert self.online_c128_mtp_pending_seq_lens is not None return self.online_c128_mtp_pending_seq_lens + def snapshot_c128_radix_state( + self, req_pool_idx: int, seq_len: int + ) -> Optional[List[torch.Tensor]]: + """Snapshot request-scoped C128 state for a radix cache node.""" + if seq_len % 128 == 0: + return None + + snapshots: List[torch.Tensor] = [] + for pool in self.compress_state_pools: + if pool is None or pool.ratio != 128: + continue + state = pool.kv_score_buffer.kv_score + if ONLINE_C128: + snapshots.append(state[req_pool_idx : req_pool_idx + 1].clone()) + else: + start = req_pool_idx * pool.ring_size + state_len = seq_len % pool.ring_size + if state_len == 0: + continue + snapshot = state.new_empty((pool.ring_size, state.shape[-1])) + half = snapshot.shape[-1] // 2 + snapshot[:, :half].zero_() + snapshot[:, half:].fill_(float("-inf")) + positions = torch.arange( + seq_len - state_len, + seq_len, + device=state.device, + dtype=torch.long, + ) + slots = positions % pool.ring_size + snapshot[slots] = state[start + slots] + snapshots.append(snapshot) + return snapshots or None + + def restore_c128_radix_state( + self, req_pool_idx: int, snapshots: Optional[List[torch.Tensor]] + ) -> None: + """Restore radix-cached C128 state into the current request slot.""" + if not snapshots: + return + + snapshot_idx = 0 + for pool in self.compress_state_pools: + if pool is None or pool.ratio != 128: + continue + snapshot = snapshots[snapshot_idx] + snapshot_idx += 1 + + state = pool.kv_score_buffer.kv_score + if ONLINE_C128: + state[req_pool_idx : req_pool_idx + 1].copy_(snapshot) + else: + start = req_pool_idx * pool.ring_size + state[start : start + pool.ring_size].copy_(snapshot) + + assert snapshot_idx == len(snapshots) + def get_indexer_compress_states(self, layer_id: int) -> CompressStatePool: self.wait_layer_transfer(layer_id) indexer_compress_state_pool = self.indexer_compress_state_pools[layer_id] diff --git a/python/sglang/srt/mem_cache/swa_radix_cache.py b/python/sglang/srt/mem_cache/swa_radix_cache.py index 42a389fbc0af..f0ce445fe2e0 100644 --- a/python/sglang/srt/mem_cache/swa_radix_cache.py +++ b/python/sglang/srt/mem_cache/swa_radix_cache.py @@ -22,7 +22,7 @@ import heapq import time from collections import defaultdict -from typing import TYPE_CHECKING, List, Optional, Tuple +from typing import TYPE_CHECKING, Dict, List, Optional, Tuple import torch from numpy import float64 @@ -80,6 +80,8 @@ def __init__(self, id: Optional[int] = None): self.host_value = None # store hash values of each page self.hash_value: Optional[List[str]] = None + # C128 state snapshots keyed by the local token offset inside this node. + self.c128_state_snapshots: Optional[Dict[int, List[torch.Tensor]]] = None # for lru list, invariant: # 1. prev has greater last_access_time @@ -386,6 +388,109 @@ def reset(self) -> None: self.swa_lru_list = LRUList(is_swa_list=True) self._record_all_cleared_event() + def _dsv4_kv_pool(self): + if self.token_to_kv_pool_allocator is None: + return None + get_kvcache = getattr(self.token_to_kv_pool_allocator, "get_kvcache", None) + if get_kvcache is None: + return None + kv_pool = get_kvcache() + if not hasattr(kv_pool, "snapshot_c128_radix_state") or not hasattr( + kv_pool, "restore_c128_radix_state" + ): + return None + return kv_pool + + def _snapshot_c128_state_for_req( + self, req: Req, seq_len: int + ) -> Optional[List[torch.Tensor]]: + kv_pool = self._dsv4_kv_pool() + if kv_pool is None or req.req_pool_idx is None or seq_len <= 0: + return None + return kv_pool.snapshot_c128_radix_state(int(req.req_pool_idx), seq_len) + + def _node_prefix_len(self, node: TreeNode) -> int: + prefix_len = 0 + while node is not None and node is not self.root_node: + prefix_len += len(node.value) + node = node.parent + return prefix_len + + def _get_c128_snapshot_at_node_end( + self, node: TreeNode + ) -> Optional[List[torch.Tensor]]: + if node.c128_state_snapshots is None: + return None + return node.c128_state_snapshots.get(len(node.value)) + + def _is_c128_restorable_node(self, node: TreeNode) -> bool: + if node is self.root_node: + return True + if self._get_c128_snapshot_at_node_end(node) is not None: + return True + return self._node_prefix_len(node) % 128 == 0 + + def _store_c128_snapshot_at_prefix_len( + self, + node: TreeNode, + prefix_len: int, + snapshot: Optional[List[torch.Tensor]], + ) -> None: + if snapshot is None or prefix_len <= 0: + return + + path: List[TreeNode] = [] + cur = node + while cur is not None and cur is not self.root_node: + path.append(cur) + cur = cur.parent + + remaining = prefix_len + for cur in reversed(path): + if remaining <= len(cur.value): + if cur.c128_state_snapshots is None: + cur.c128_state_snapshots = {} + cur.c128_state_snapshots[remaining] = snapshot + return + remaining -= len(cur.value) + + def _store_c128_partial_snapshot_for_req( + self, req: Req, node: TreeNode, prefix_len: int + ) -> None: + if prefix_len % 128 == 0: + return + snapshot = self._snapshot_c128_state_for_req(req, prefix_len) + self._store_c128_snapshot_at_prefix_len(node, prefix_len, snapshot) + + def _find_c128_restorable_node(self, node: TreeNode) -> TreeNode: + if self._dsv4_kv_pool() is None: + return node + while node is not self.root_node and not self._is_c128_restorable_node(node): + prefix_len = self._node_prefix_len(node) + aligned_prefix_len = prefix_len // 128 * 128 + parent_prefix_len = prefix_len - len(node.value) + if aligned_prefix_len > parent_prefix_len: + split_len = aligned_prefix_len - parent_prefix_len + return self._split_node(node.key, node, split_len) + node = node.parent + return node + + def restore_c128_state_for_reqs(self, reqs: List[Req]) -> None: + kv_pool = self._dsv4_kv_pool() + if kv_pool is None: + return + for req in reqs: + if req.req_pool_idx is None: + continue + node = getattr(req, "last_node", self.root_node) + prefix_len = len(getattr(req, "prefix_indices", [])) + snapshot = self._get_c128_snapshot_at_node_end(node) + if node is self.root_node or not self._is_c128_restorable_node(node): + continue + assert self._node_prefix_len(node) == prefix_len + if snapshot is not None: + kv_pool.restore_c128_radix_state(int(req.req_pool_idx), snapshot) + def match_prefix(self, params: MatchPrefixParams) -> MatchResult: """Find the matching prefix from the radix tree. Args: @@ -468,6 +573,12 @@ def cache_finished_req(self, req: Req, is_insert: bool = True) -> None: swa_evicted_seqlen=req.swa_evicted_seqlen, ) ) + match_result = self.match_prefix(MatchPrefixParams(key=radix_key)) + if len(match_result.device_indices) == page_aligned_len: + last_node = match_result.last_device_node + self._store_c128_partial_snapshot_for_req( + req, last_node, page_aligned_len + ) else: self.token_to_kv_pool_allocator.free( kv_indices[old_prefix_len:page_aligned_len] @@ -551,6 +662,7 @@ def cache_unfinished_req(self, req: Req, chunked=False) -> None: req.prefix_indices = new_indices req.last_node = new_last_node req.swa_uuid_for_lock = swa_uuid_for_lock + self._store_c128_partial_snapshot_for_req(req, new_last_node, len(new_indices)) def pretty_print(self) -> None: self._print_helper(self.root_node, 0) @@ -960,6 +1072,12 @@ def _match_post_processor( else: value = torch.empty((0,), dtype=torch.int64, device=self.device) + if params.req is not None and len(value) > 0: + last_node = self._find_c128_restorable_node(last_node) + c128_prefix_len = self._node_prefix_len(last_node) + if c128_prefix_len < len(value): + value = value[:c128_prefix_len] + return MatchResult( device_indices=value, last_device_node=last_node, @@ -997,6 +1115,12 @@ def _compact_single_child_chain(self, node: TreeNode) -> None: is_bigram=node.key.is_bigram, ) node.value = torch.cat([node.value, child.value]) + if child.c128_state_snapshots is not None: + offset = len(node.value) - len(child.value) + if node.c128_state_snapshots is None: + node.c128_state_snapshots = {} + for rel_len, snapshot in child.c128_state_snapshots.items(): + node.c128_state_snapshots[offset + rel_len] = snapshot node.children = child.children for grandchild in node.children.values(): grandchild.parent = node @@ -1075,6 +1199,19 @@ def _split_node(self, key: RadixKey, child: TreeNode, split_len: int) -> TreeNod child.key = child.key[split_len:] assert len(child.key) > 0, f"child.key should not be empty" child.value = child.value[split_len:].clone() + if child.c128_state_snapshots is not None: + parent_snapshots = { + rel_len: snapshot + for rel_len, snapshot in child.c128_state_snapshots.items() + if rel_len <= split_len + } + child_snapshots = { + rel_len - split_len: snapshot + for rel_len, snapshot in child.c128_state_snapshots.items() + if rel_len > split_len + } + new_node.c128_state_snapshots = parent_snapshots or None + child.c128_state_snapshots = child_snapshots or None new_node.parent.children[key.child_key(self.page_size)] = new_node new_node.hash_value, child.hash_value = split_node_hash_value( child.hash_value, split_len, self.page_size diff --git a/test/registered/unit/disaggregation/test_disaggregation_wire.py b/test/registered/unit/disaggregation/test_disaggregation_wire.py index 76ee5874b42d..31fdcd039da1 100644 --- a/test/registered/unit/disaggregation/test_disaggregation_wire.py +++ b/test/registered/unit/disaggregation/test_disaggregation_wire.py @@ -9,6 +9,7 @@ unpack_int_lists, unpack_list_of_buffers, ) +from sglang.srt.disaggregation.utils import get_dsv4_c128_state_indices from sglang.test.ci.ci_register import register_cpu_ci register_cpu_ci(est_time=2, suite="base-a-test-cpu") @@ -90,5 +91,25 @@ def test_mismatched_nonempty_lengths_raise(self): group_concurrent_contiguous(self._arr([1, 2, 3]), self._arr([1, 2])) +class TestDSV4C128StateIndices(unittest.TestCase): + def test_online_uses_request_slot(self): + np.testing.assert_array_equal( + get_dsv4_c128_state_indices(7, 256, online=True, ring_size=1), + np.array([7], dtype=np.int32), + ) + + def test_offline_aligned_boundary_has_no_partial_state(self): + np.testing.assert_array_equal( + get_dsv4_c128_state_indices(7, 256, online=False, ring_size=128), + np.empty((0,), dtype=np.int32), + ) + + def test_offline_partial_boundary_uses_request_local_page(self): + np.testing.assert_array_equal( + get_dsv4_c128_state_indices(7, 129, online=False, ring_size=256), + np.array([15], dtype=np.int32), + ) + + if __name__ == "__main__": unittest.main() diff --git a/test/registered/unit/mem_cache/test_dsv4_c128_radix_state.py b/test/registered/unit/mem_cache/test_dsv4_c128_radix_state.py new file mode 100644 index 000000000000..f7b3e9b4be65 --- /dev/null +++ b/test/registered/unit/mem_cache/test_dsv4_c128_radix_state.py @@ -0,0 +1,177 @@ +import unittest +from array import array + +import torch + +from sglang.srt.mem_cache.allocator.swa import SWATokenToKVPoolAllocator +from sglang.srt.mem_cache.base_prefix_cache import InsertParams, MatchPrefixParams +from sglang.srt.mem_cache.base_swa_memory_pool import BaseSWAKVPool +from sglang.srt.mem_cache.cache_init_params import CacheInitParams +from sglang.srt.mem_cache.memory_pool import ReqToTokenPool +from sglang.srt.mem_cache.swa_radix_cache import RadixKey, SWARadixCache +from sglang.test.ci.ci_register import register_cpu_ci + +register_cpu_ci(est_time=5, suite="base-b-test-cpu") + + +class FakeDSV4KVPool(BaseSWAKVPool): + def __init__(self): + self.full_kv_pool = None + self.swa_kv_pool = None + self.restored = [] + self.snapshotted = [] + + def get_key_buffer(self, layer_id: int) -> torch.Tensor: + raise NotImplementedError + + def get_value_buffer(self, layer_id: int) -> torch.Tensor: + raise NotImplementedError + + def get_kv_buffer(self, layer_id: int): + raise NotImplementedError + + def set_kv_buffer(self, layer, loc, cache_k, cache_v) -> None: + raise NotImplementedError + + def register_mapping(self, full_to_swa_index_mapping: torch.Tensor) -> None: + pass + + def translate_loc_from_full_to_swa(self, kv_indices: torch.Tensor) -> torch.Tensor: + return kv_indices + + def get_state_buf_infos(self): + return [], [], [] + + def snapshot_c128_radix_state(self, req_pool_idx: int, seq_len: int): + self.snapshotted.append((req_pool_idx, seq_len)) + return [torch.tensor([req_pool_idx, seq_len], dtype=torch.int64)] + + def restore_c128_radix_state(self, req_pool_idx: int, snapshots): + self.restored.append((req_pool_idx, snapshots)) + + +class FakeReq: + def __init__(self, req_pool_idx: int): + self.req_pool_idx = req_pool_idx + self.prefix_indices = [] + self.last_node = None + + +class TestDSV4C128RadixState(unittest.TestCase): + def _make_cache(self): + kv_pool = FakeDSV4KVPool() + allocator = SWATokenToKVPoolAllocator( + size=512, + size_swa=512, + page_size=1, + dtype=torch.float16, + device="cpu", + kvcache=kv_pool, + need_sort=False, + ) + req_to_token_pool = ReqToTokenPool( + size=4, max_context_len=512, device="cpu", enable_memory_saver=False + ) + cache = SWARadixCache( + CacheInitParams( + disable=False, + req_to_token_pool=req_to_token_pool, + token_to_kv_pool_allocator=allocator, + page_size=1, + sliding_window_size=512, + ) + ) + return cache, kv_pool + + def _insert_tokens(self, cache: SWARadixCache, length: int): + key = RadixKey(array("q", range(length))) + value = torch.arange(length, dtype=torch.int64) + cache.insert(InsertParams(key=key, value=value)) + match = cache.match_prefix(MatchPrefixParams(key=key)) + self.assertEqual(len(match.device_indices), length) + return match.last_device_node + + def test_aligned_c128_boundary_does_not_need_snapshot(self): + cache, kv_pool = self._make_cache() + self._insert_tokens(cache, 260) + + req = FakeReq(req_pool_idx=2) + match = cache.match_prefix( + MatchPrefixParams(key=RadixKey(array("q", range(128))), req=req) + ) + + self.assertEqual(len(match.device_indices), 128) + self.assertIsNone(match.last_device_node.c128_state_snapshots) + + req.prefix_indices = match.device_indices + req.last_node = match.last_device_node + cache.restore_c128_state_for_reqs([req]) + self.assertEqual(kv_pool.restored, []) + + def test_partial_c128_snapshot_moves_to_split_node_and_restores(self): + cache, kv_pool = self._make_cache() + leaf = self._insert_tokens(cache, 260) + + snapshot = [torch.tensor([7, 130], dtype=torch.int64)] + cache._store_c128_snapshot_at_prefix_len(leaf, 130, snapshot) + + req = FakeReq(req_pool_idx=3) + match = cache.match_prefix( + MatchPrefixParams(key=RadixKey(array("q", range(130))), req=req) + ) + + self.assertEqual(len(match.device_indices), 130) + self.assertIs( + match.last_device_node.c128_state_snapshots[130], + snapshot, + ) + + req.prefix_indices = match.device_indices + req.last_node = match.last_device_node + cache.restore_c128_state_for_reqs([req]) + + self.assertEqual(len(kv_pool.restored), 1) + restored_req_pool_idx, restored_snapshot = kv_pool.restored[0] + self.assertEqual(restored_req_pool_idx, 3) + self.assertIs(restored_snapshot, snapshot) + + def test_missing_partial_snapshot_falls_back_to_aligned_boundary(self): + cache, kv_pool = self._make_cache() + self._insert_tokens(cache, 260) + + req = FakeReq(req_pool_idx=3) + match = cache.match_prefix( + MatchPrefixParams(key=RadixKey(array("q", range(130))), req=req) + ) + + self.assertEqual(len(match.device_indices), 128) + self.assertEqual(cache._node_prefix_len(match.last_device_node), 128) + + req.prefix_indices = match.device_indices + req.last_node = match.last_device_node + cache.restore_c128_state_for_reqs([req]) + self.assertEqual(kv_pool.restored, []) + + def test_store_partial_snapshot_uses_requested_prefix_length(self): + cache, kv_pool = self._make_cache() + leaf = self._insert_tokens(cache, 260) + req = FakeReq(req_pool_idx=4) + + cache._store_c128_partial_snapshot_for_req(req, leaf, 130) + + self.assertEqual(kv_pool.snapshotted, [(4, 130)]) + self.assertIn(130, leaf.c128_state_snapshots) + + def test_store_aligned_boundary_skips_snapshot(self): + cache, kv_pool = self._make_cache() + leaf = self._insert_tokens(cache, 260) + req = FakeReq(req_pool_idx=4) + + cache._store_c128_partial_snapshot_for_req(req, leaf, 128) + + self.assertEqual(kv_pool.snapshotted, []) + self.assertIsNone(leaf.c128_state_snapshots) + + +if __name__ == "__main__": + unittest.main() From 877086cb8b15370f86a01770a95b63fcfb2b63d4 Mon Sep 17 00:00:00 2001 From: zhangxiaolei Date: Mon, 22 Jun 2026 23:45:18 +0800 Subject: [PATCH 12/24] Fix dsv4 c128 offline pd sync pr 601 (#610) --- python/sglang/srt/disaggregation/base/conn.py | 4 ++ .../sglang/srt/disaggregation/common/conn.py | 8 ++++ python/sglang/srt/disaggregation/decode.py | 7 +++ python/sglang/srt/disaggregation/fake/conn.py | 1 + .../srt/disaggregation/mooncake/conn.py | 24 +++++++++++ python/sglang/srt/disaggregation/mori/conn.py | 26 +++++++++++ python/sglang/srt/disaggregation/nixl/conn.py | 19 ++++++++ python/sglang/srt/disaggregation/prefill.py | 17 ++++++-- python/sglang/srt/disaggregation/utils.py | 4 +- .../srt/mem_cache/deepseek_v4_memory_pool.py | 21 ++++++++- .../sglang/srt/mem_cache/swa_radix_cache.py | 8 +++- .../test_disaggregation_wire.py | 8 +++- .../mem_cache/test_dsv4_c128_radix_state.py | 43 +++++++++++++++++++ 13 files changed, 180 insertions(+), 10 deletions(-) diff --git a/python/sglang/srt/disaggregation/base/conn.py b/python/sglang/srt/disaggregation/base/conn.py index e998e16bcceb..67505e955479 100644 --- a/python/sglang/srt/disaggregation/base/conn.py +++ b/python/sglang/srt/disaggregation/base/conn.py @@ -134,6 +134,9 @@ def send( def pop_decode_prefix_len(self) -> int: return 0 + def pop_transfer_input_len(self) -> Optional[int]: + return None + def should_send_kv_chunk(self, num_pages: int, last_chunk: bool) -> bool: return num_pages > 0 @@ -184,6 +187,7 @@ def send_metadata( aux_index: Optional[int] = None, state_indices: Optional[List] = None, decode_prefix_len: Optional[int] = None, + transfer_input_len: Optional[int] = None, ): """ Notify the prefill server about the kv indices, aux index, and state_indices. diff --git a/python/sglang/srt/disaggregation/common/conn.py b/python/sglang/srt/disaggregation/common/conn.py index 8af3a2b90c95..e7b54cccfa7b 100644 --- a/python/sglang/srt/disaggregation/common/conn.py +++ b/python/sglang/srt/disaggregation/common/conn.py @@ -175,6 +175,7 @@ def __init__( self.register_to_bootstrap() self.transfer_infos = {} self.req_to_decode_prefix_len: Dict[int, int] = {} + self.req_to_transfer_input_len: Dict[int, int] = {} self.decode_kv_args_table = {} self.pp_group = get_pp_group() # If a timeout happens on the prefill side, it means prefill instances @@ -838,6 +839,9 @@ def init(self, num_kv_indices: int, aux_index: Optional[int] = None): def pop_decode_prefix_len(self) -> int: return self.kv_mgr.req_to_decode_prefix_len.pop(self.bootstrap_room, 0) + def pop_transfer_input_len(self) -> Optional[int]: + return self.kv_mgr.req_to_transfer_input_len.pop(self.bootstrap_room, None) + def should_send_kv_chunk(self, num_pages: int, last_chunk: bool) -> bool: return num_pages > 0 or last_chunk @@ -926,6 +930,8 @@ def clear(self) -> None: self.kv_mgr.request_status.pop(self.bootstrap_room, None) if hasattr(self.kv_mgr, "req_to_decode_prefix_len"): self.kv_mgr.req_to_decode_prefix_len.pop(self.bootstrap_room, None) + if hasattr(self.kv_mgr, "req_to_transfer_input_len"): + self.kv_mgr.req_to_transfer_input_len.pop(self.bootstrap_room, None) if hasattr(self.kv_mgr, "transfer_infos"): self.kv_mgr.transfer_infos.pop(self.bootstrap_room, None) @@ -1136,6 +1142,8 @@ def send_metadata( kv_indices: npt.NDArray[np.int32], aux_index: Optional[int] = None, state_indices: Optional[List[int]] = None, + decode_prefix_len: Optional[int] = None, + transfer_input_len: Optional[int] = None, ): raise NotImplementedError diff --git a/python/sglang/srt/disaggregation/decode.py b/python/sglang/srt/disaggregation/decode.py index 15c315001293..5ae0ef11a65e 100644 --- a/python/sglang/srt/disaggregation/decode.py +++ b/python/sglang/srt/disaggregation/decode.py @@ -1053,6 +1053,12 @@ def _c128_state_payload(): state_types = self.kv_manager.kv_args.state_types state_indices: Optional[List] = [] + if StateType.C128_STATE in state_types: + clear_c128_state = getattr( + self.token_to_kv_pool, "clear_c128_radix_state", None + ) + if clear_c128_state is not None: + clear_c128_state(int(decode_req.req.req_pool_idx)) for st in state_types: if st == StateType.MAMBA: state_indices.append(_mamba_payload()) @@ -1077,6 +1083,7 @@ def _c128_state_payload(): decode_req.metadata_buffer_index, state_indices, decode_prefix_len=total_prefix_len, + transfer_input_len=seq_len, ) if ( self.transfer_queue.enable_staging diff --git a/python/sglang/srt/disaggregation/fake/conn.py b/python/sglang/srt/disaggregation/fake/conn.py index 09c5f80ce492..02373f192573 100644 --- a/python/sglang/srt/disaggregation/fake/conn.py +++ b/python/sglang/srt/disaggregation/fake/conn.py @@ -125,6 +125,7 @@ def send_metadata( aux_index: Optional[int] = None, state_indices: Optional[List] = None, decode_prefix_len: Optional[int] = None, + transfer_input_len: Optional[int] = None, ): self.has_sent_metadata = True logger.debug( diff --git a/python/sglang/srt/disaggregation/mooncake/conn.py b/python/sglang/srt/disaggregation/mooncake/conn.py index ab583b90dd52..ccada456639b 100644 --- a/python/sglang/srt/disaggregation/mooncake/conn.py +++ b/python/sglang/srt/disaggregation/mooncake/conn.py @@ -76,6 +76,7 @@ class TransferInfo: required_dst_info_num: int is_dummy: bool decode_prefix_len: Optional[int] = None + transfer_input_len: Optional[int] = None # Note: always put the optional staging field at the final (it will be set through 'STAGING_RSP' pkg when needed) staging: Optional[StagingTransferInfo] = None @@ -104,6 +105,9 @@ def from_zmq(cls, msg: List[bytes]): decode_prefix_len=( int(msg[8].decode("ascii")) if len(msg) > 8 and msg[8] != b"" else None ), + transfer_input_len=( + int(msg[9].decode("ascii")) if len(msg) > 9 and msg[9] != b"" else None + ), ) @@ -1014,6 +1018,13 @@ def maybe_send_extra( ) src_indices = list(indices) dst_indices_local = list(dst_indices) + if st == StateType.C128_STATE and len(dst_indices_local) == 0: + # Decode decides whether a request has a partial C128 state + # to receive. At 128-aligned boundaries it intentionally + # registers no C128 dst row and clears the request-scoped + # state locally; a stale prefill-side source row must not + # keep the transfer thread alive or corrupt the next request. + continue if len(src_indices) != len(dst_indices_local): # These components are position- or request-indexed: # truncating silently misaligns rows and corrupts KV. @@ -1391,6 +1402,7 @@ def transfer_worker( if kv_chunk.room in self.transfer_infos: self.transfer_infos.pop(kv_chunk.room) self.req_to_decode_prefix_len.pop(kv_chunk.room, None) + self.req_to_transfer_input_len.pop(kv_chunk.room, None) except Exception as e: # NOTE(shangming): Remove this when we make sure the transfer thread is bug-free @@ -1492,6 +1504,16 @@ def bootstrap_thread(): ), 0, ) + transfer_input_len = next( + ( + info.transfer_input_len + for info in self.transfer_infos[room].values() + if info.transfer_input_len is not None + ), + None, + ) + if transfer_input_len is not None: + self.req_to_transfer_input_len[room] = transfer_input_len self.update_status(room, KVPoll.WaitingForInput) threading.Thread(target=bootstrap_thread).start() @@ -1837,6 +1859,7 @@ def send_metadata( aux_index: Optional[int] = None, state_indices: Optional[List] = None, decode_prefix_len: Optional[int] = None, + transfer_input_len: Optional[int] = None, ): if self.bootstrap_infos is None: self.kv_mgr.record_failure( @@ -1875,6 +1898,7 @@ def send_metadata( ), str(self.required_dst_info_num).encode("ascii"), str(decode_prefix_len or 0).encode("ascii"), + str(transfer_input_len or 0).encode("ascii"), ] ) self.init_time = time.time() diff --git a/python/sglang/srt/disaggregation/mori/conn.py b/python/sglang/srt/disaggregation/mori/conn.py index a7063ff3dc78..da1bdc5d61a5 100644 --- a/python/sglang/srt/disaggregation/mori/conn.py +++ b/python/sglang/srt/disaggregation/mori/conn.py @@ -124,6 +124,7 @@ class TransferInfo: # populate this field (older receiver or radix-cache feature off) -> treat # as 0 (no prefix hit, full send) for backward compatibility. decode_prefix_len: Optional[int] = None + transfer_input_len: Optional[int] = None @classmethod def from_zmq(cls, payload: List[bytes]) -> TransferInfo: @@ -156,6 +157,11 @@ def from_zmq(cls, payload: List[bytes]) -> TransferInfo: else: decode_prefix_len = None + if len(payload) > 9 and payload[9]: + transfer_input_len: Optional[int] = int(payload[9].decode("ascii")) + else: + transfer_input_len = None + # A transfer is "dummy" only when the receiver does not need any # kv/aux/state delivered. When decode_prefix_len > 0 and the delta is # exactly zero (full prefix hit), dst_kv_indices is empty but aux is @@ -174,6 +180,7 @@ def from_zmq(cls, payload: List[bytes]) -> TransferInfo: required_dst_info_num=required_dst_info_num, is_dummy=is_dummy, decode_prefix_len=decode_prefix_len, + transfer_input_len=transfer_input_len, ) @@ -521,6 +528,18 @@ def _handle_transfer_message(self, payload: List[bytes]) -> None: self.req_to_decode_prefix_len[transfer_info.room] = ( chosen_prefix_len ) + transfer_input_len = next( + ( + info.transfer_input_len + for info in infos.values() + if info.transfer_input_len is not None + ), + None, + ) + if transfer_input_len is not None: + self.req_to_transfer_input_len[transfer_info.room] = ( + transfer_input_len + ) if chosen_prefix_len > 0: # Surface incremental KV transfer at INFO so it's # visible without bumping the global log level. @@ -1670,6 +1689,7 @@ def send_metadata( aux_index: Optional[int] = None, state_indices: Optional[List] = None, decode_prefix_len: Optional[int] = None, + transfer_input_len: Optional[int] = None, ): if self.bootstrap_infos is None or self.bootstrap_room is None: return @@ -1685,6 +1705,11 @@ def send_metadata( if decode_prefix_len is not None and decode_prefix_len > 0 else b"" ) + transfer_input_len_bytes = ( + str(int(transfer_input_len)).encode("ascii") + if transfer_input_len is not None and transfer_input_len > 0 + else b"" + ) for bootstrap_info in self.bootstrap_infos: sock, lock = self._connect_to_bootstrap_server(bootstrap_info) @@ -1706,6 +1731,7 @@ def send_metadata( state_bytes, str(self.required_dst_info_num).encode("ascii"), decode_prefix_bytes, + transfer_input_len_bytes, ] ) self.init_time = time.time() diff --git a/python/sglang/srt/disaggregation/nixl/conn.py b/python/sglang/srt/disaggregation/nixl/conn.py index c9ddbd382a47..5b9b411efda9 100644 --- a/python/sglang/srt/disaggregation/nixl/conn.py +++ b/python/sglang/srt/disaggregation/nixl/conn.py @@ -69,6 +69,7 @@ class TransferInfo: required_dst_info_num: int dst_state_indices: List[List[int]] decode_prefix_len: Optional[int] = None # for decode radix cache + transfer_input_len: Optional[int] = None # NOTE: optional staging field; populated via STAGING_RSP. Keep at the # end so positional construction in from_zmq() continues to work. staging: Optional[StagingTransferInfo] = None @@ -100,6 +101,9 @@ def from_zmq(cls, msg: List[bytes]): decode_prefix_len=( int(msg[8].decode("ascii")) if len(msg) > 8 and msg[8] != b"" else None ), # hacky just add it into the message that will be sent + transfer_input_len=( + int(msg[9].decode("ascii")) if len(msg) > 9 and msg[9] != b"" else None + ), ) @@ -870,6 +874,7 @@ def transfer_worker(self, queue: FastQueue, staging_buffer=None): # transfer_worker; staging prefetch sets are NIXL-only). self.transfer_infos.pop(room, None) self.req_to_decode_prefix_len.pop(room, None) + self.req_to_transfer_input_len.pop(room, None) if self.enable_staging and self._staging_ctx is not None: self._staging_ctx.prefetched_rooms.discard(room) self._staging_ctx.prefetch_requested = { @@ -1623,6 +1628,8 @@ def maybe_send_extra( raise RuntimeError( f"PD Disaggregation does NOT support PD different TP sizes for non-MLA {st.upper()} hybrid models yet." ) + if st == StateType.C128_STATE and len(dst_indices) == 0: + continue if len(src_indices) != len(dst_indices): raise RuntimeError( f"State index length mismatch at component {i}: " @@ -1885,6 +1892,16 @@ def bootstrap_thread(): ), 0, ) + transfer_input_len = next( + ( + info.transfer_input_len + for info in self.transfer_infos[room].values() + if info.transfer_input_len is not None + ), + None, + ) + if transfer_input_len is not None: + self.req_to_transfer_input_len[room] = transfer_input_len logger.debug(f"{room=} is bootstrapped") self.update_status(room, KVPoll.WaitingForInput) @@ -2006,6 +2023,7 @@ def send_metadata( aux_index: Optional[int] = None, state_indices: Optional[List] = None, decode_prefix_len: Optional[int] = None, + transfer_input_len: Optional[int] = None, ): if self.bootstrap_infos is None: logger.error( @@ -2053,6 +2071,7 @@ def send_metadata( str(self.required_dst_info_num).encode("ascii"), packed_state_indices, str(decode_prefix_len or 0).encode("ascii"), + str(transfer_input_len or 0).encode("ascii"), ] ) diff --git a/python/sglang/srt/disaggregation/prefill.py b/python/sglang/srt/disaggregation/prefill.py index 8814603fff01..c9e544e1fd97 100644 --- a/python/sglang/srt/disaggregation/prefill.py +++ b/python/sglang/srt/disaggregation/prefill.py @@ -248,6 +248,7 @@ def create_sender(self, req: Req, num_kv_heads: int) -> bool: dest_tp_ranks=dest_tp_ranks, pp_rank=self.pp_rank, ) + req.disagg_transfer_input_len = len(req.origin_input_ids) self._process_req(req) req.pending_bootstrap = True return True @@ -270,9 +271,14 @@ def finalize_bootstrap(self, req: Req) -> bool: return False req.time_stats.set_bootstrap_done_time() - num_kv_indices = len(req.origin_input_ids) - decode_prefix_len = req.disagg_kv_sender.pop_decode_prefix_len() + transfer_input_len = req.disagg_kv_sender.pop_transfer_input_len() + if transfer_input_len is None: + transfer_input_len = getattr( + req, "disagg_transfer_input_len", len(req.origin_input_ids) + ) + req.disagg_transfer_input_len = transfer_input_len + num_kv_indices = transfer_input_len req.start_send_idx = decode_prefix_len num_kv_indices_to_send = num_kv_indices - decode_prefix_len num_pages = kv_to_page_num( @@ -967,10 +973,13 @@ def send_kv_chunk( """ page_size = self.token_to_kv_pool_allocator.page_size start_idx = req.start_send_idx + transfer_input_len = getattr( + req, "disagg_transfer_input_len", len(req.origin_input_ids) + ) end_idx = ( end_idx if end_idx is not None - else min(req.fill_len, len(req.origin_input_ids)) + else min(req.fill_len, transfer_input_len) ) if not last_chunk: @@ -1001,7 +1010,7 @@ def send_kv_chunk( # length here avoids emitting an extra state page when the sampled # token crosses a page boundary, which mismatched src/dst lengths in # group_concurrent_contiguous. - seq_len = min(req.fill_len, len(req.origin_input_ids)) + seq_len = min(req.fill_len, transfer_input_len) def _mamba_payload(): return [ diff --git a/python/sglang/srt/disaggregation/utils.py b/python/sglang/srt/disaggregation/utils.py index cc98b3b5b908..b2315b31ac81 100644 --- a/python/sglang/srt/disaggregation/utils.py +++ b/python/sglang/srt/disaggregation/utils.py @@ -40,10 +40,10 @@ def get_dsv4_c128_state_indices( ring_size: int, ) -> np.ndarray: """Return the PD transfer row/page indices for DSV4 C128 state.""" - if online: - return np.array([int(req_pool_idx)], dtype=np.int32) if seq_len == 0 or seq_len % 128 == 0: return np.empty((0,), dtype=np.int32) + if online: + return np.array([int(req_pool_idx)], dtype=np.int32) assert ring_size % 128 == 0, f"C128 ring_size must be 128-aligned, got {ring_size}" pages_per_req = ring_size // 128 diff --git a/python/sglang/srt/mem_cache/deepseek_v4_memory_pool.py b/python/sglang/srt/mem_cache/deepseek_v4_memory_pool.py index d0f363e3d87d..0cf07c09c402 100644 --- a/python/sglang/srt/mem_cache/deepseek_v4_memory_pool.py +++ b/python/sglang/srt/mem_cache/deepseek_v4_memory_pool.py @@ -958,7 +958,7 @@ def snapshot_c128_radix_state( snapshots.append(state[req_pool_idx : req_pool_idx + 1].clone()) else: start = req_pool_idx * pool.ring_size - state_len = seq_len % pool.ring_size + state_len = seq_len % 128 if state_len == 0: continue snapshot = state.new_empty((pool.ring_size, state.shape[-1])) @@ -999,6 +999,25 @@ def restore_c128_radix_state( assert snapshot_idx == len(snapshots) + def clear_c128_radix_state(self, req_pool_idx: int) -> None: + """Reset request-scoped C128 state at a radix-restorable boundary.""" + for pool in self.compress_state_pools: + if pool is None or pool.ratio != 128: + continue + + state = pool.kv_score_buffer.kv_score + if ONLINE_C128: + row = state[req_pool_idx] + head_dim = row.shape[-1] // 3 + row[:head_dim].fill_(float("-inf")) + row[head_dim:].zero_() + else: + start = req_pool_idx * pool.ring_size + rows = state[start : start + pool.ring_size] + half = rows.shape[-1] // 2 + rows[:, :half].zero_() + rows[:, half:].fill_(float("-inf")) + def get_indexer_compress_states(self, layer_id: int) -> CompressStatePool: self.wait_layer_transfer(layer_id) indexer_compress_state_pool = self.indexer_compress_state_pools[layer_id] diff --git a/python/sglang/srt/mem_cache/swa_radix_cache.py b/python/sglang/srt/mem_cache/swa_radix_cache.py index f0ce445fe2e0..d442a1bd01cb 100644 --- a/python/sglang/srt/mem_cache/swa_radix_cache.py +++ b/python/sglang/srt/mem_cache/swa_radix_cache.py @@ -395,8 +395,10 @@ def _dsv4_kv_pool(self): if get_kvcache is None: return None kv_pool = get_kvcache() - if not hasattr(kv_pool, "snapshot_c128_radix_state") or not hasattr( - kv_pool, "restore_c128_radix_state" + if ( + not hasattr(kv_pool, "snapshot_c128_radix_state") + or not hasattr(kv_pool, "restore_c128_radix_state") + or not hasattr(kv_pool, "clear_c128_radix_state") ): return None return kv_pool @@ -490,6 +492,8 @@ def restore_c128_state_for_reqs(self, reqs: List[Req]) -> None: assert self._node_prefix_len(node) == prefix_len if snapshot is not None: kv_pool.restore_c128_radix_state(int(req.req_pool_idx), snapshot) + else: + kv_pool.clear_c128_radix_state(int(req.req_pool_idx)) def match_prefix(self, params: MatchPrefixParams) -> MatchResult: """Find the matching prefix from the radix tree. diff --git a/test/registered/unit/disaggregation/test_disaggregation_wire.py b/test/registered/unit/disaggregation/test_disaggregation_wire.py index 31fdcd039da1..76208a5cc085 100644 --- a/test/registered/unit/disaggregation/test_disaggregation_wire.py +++ b/test/registered/unit/disaggregation/test_disaggregation_wire.py @@ -92,9 +92,15 @@ def test_mismatched_nonempty_lengths_raise(self): class TestDSV4C128StateIndices(unittest.TestCase): - def test_online_uses_request_slot(self): + def test_online_aligned_boundary_has_no_partial_state(self): np.testing.assert_array_equal( get_dsv4_c128_state_indices(7, 256, online=True, ring_size=1), + np.empty((0,), dtype=np.int32), + ) + + def test_online_partial_boundary_uses_request_slot(self): + np.testing.assert_array_equal( + get_dsv4_c128_state_indices(7, 257, online=True, ring_size=1), np.array([7], dtype=np.int32), ) diff --git a/test/registered/unit/mem_cache/test_dsv4_c128_radix_state.py b/test/registered/unit/mem_cache/test_dsv4_c128_radix_state.py index f7b3e9b4be65..fe54ca703d2d 100644 --- a/test/registered/unit/mem_cache/test_dsv4_c128_radix_state.py +++ b/test/registered/unit/mem_cache/test_dsv4_c128_radix_state.py @@ -3,6 +3,7 @@ import torch +from sglang.srt.mem_cache import deepseek_v4_memory_pool as dsv4_memory_pool from sglang.srt.mem_cache.allocator.swa import SWATokenToKVPoolAllocator from sglang.srt.mem_cache.base_prefix_cache import InsertParams, MatchPrefixParams from sglang.srt.mem_cache.base_swa_memory_pool import BaseSWAKVPool @@ -19,6 +20,7 @@ def __init__(self): self.full_kv_pool = None self.swa_kv_pool = None self.restored = [] + self.cleared = [] self.snapshotted = [] def get_key_buffer(self, layer_id: int) -> torch.Tensor: @@ -49,6 +51,9 @@ def snapshot_c128_radix_state(self, req_pool_idx: int, seq_len: int): def restore_c128_radix_state(self, req_pool_idx: int, snapshots): self.restored.append((req_pool_idx, snapshots)) + def clear_c128_radix_state(self, req_pool_idx: int): + self.cleared.append(req_pool_idx) + class FakeReq: def __init__(self, req_pool_idx: int): @@ -57,6 +62,19 @@ def __init__(self, req_pool_idx: int): self.last_node = None +class FakeKVScoreBuffer: + def __init__(self, kv_score: torch.Tensor): + self.kv_score = kv_score + + +class FakeCompressStatePool: + ratio = 128 + + def __init__(self, ring_size: int, kv_score: torch.Tensor): + self.ring_size = ring_size + self.kv_score_buffer = FakeKVScoreBuffer(kv_score) + + class TestDSV4C128RadixState(unittest.TestCase): def _make_cache(self): kv_pool = FakeDSV4KVPool() @@ -107,6 +125,7 @@ def test_aligned_c128_boundary_does_not_need_snapshot(self): req.last_node = match.last_device_node cache.restore_c128_state_for_reqs([req]) self.assertEqual(kv_pool.restored, []) + self.assertEqual(kv_pool.cleared, [2]) def test_partial_c128_snapshot_moves_to_split_node_and_restores(self): cache, kv_pool = self._make_cache() @@ -151,6 +170,7 @@ def test_missing_partial_snapshot_falls_back_to_aligned_boundary(self): req.last_node = match.last_device_node cache.restore_c128_state_for_reqs([req]) self.assertEqual(kv_pool.restored, []) + self.assertEqual(kv_pool.cleared, [3]) def test_store_partial_snapshot_uses_requested_prefix_length(self): cache, kv_pool = self._make_cache() @@ -172,6 +192,29 @@ def test_store_aligned_boundary_skips_snapshot(self): self.assertEqual(kv_pool.snapshotted, []) self.assertIsNone(leaf.c128_state_snapshots) + def test_offline_snapshot_only_copies_current_c128_block(self): + kv_score = torch.arange(512 * 4, dtype=torch.float32).reshape(512, 4) + kv_pool = dsv4_memory_pool.DeepSeekV4TokenToKVPool.__new__( + dsv4_memory_pool.DeepSeekV4TokenToKVPool + ) + kv_pool.compress_state_pools = [FakeCompressStatePool(256, kv_score)] + + old_online_c128 = dsv4_memory_pool.ONLINE_C128 + dsv4_memory_pool.ONLINE_C128 = False + try: + snapshot = kv_pool.snapshot_c128_radix_state(req_pool_idx=1, seq_len=130)[ + 0 + ] + finally: + dsv4_memory_pool.ONLINE_C128 = old_online_c128 + + start = 256 + expected = kv_score.new_empty((256, 4)) + expected[:, :2].zero_() + expected[:, 2:].fill_(float("-inf")) + expected[128:130] = kv_score[start + 128 : start + 130] + torch.testing.assert_close(snapshot, expected) + if __name__ == "__main__": unittest.main() From 5bd49054731ee4e10f329098bc0d2847df4b043f Mon Sep 17 00:00:00 2001 From: zhangxiaolei Date: Tue, 23 Jun 2026 01:10:45 +0800 Subject: [PATCH 13/24] Fix offline C128 speculative draft cleanup (#611) --- .../srt/mem_cache/deepseek_v4_memory_pool.py | 31 +++++++++++++++++++ .../sglang/srt/speculative/eagle_worker_v2.py | 12 +++++++ 2 files changed, 43 insertions(+) diff --git a/python/sglang/srt/mem_cache/deepseek_v4_memory_pool.py b/python/sglang/srt/mem_cache/deepseek_v4_memory_pool.py index 0cf07c09c402..d1b298492950 100644 --- a/python/sglang/srt/mem_cache/deepseek_v4_memory_pool.py +++ b/python/sglang/srt/mem_cache/deepseek_v4_memory_pool.py @@ -1018,6 +1018,37 @@ def clear_c128_radix_state(self, req_pool_idx: int) -> None: rows[:, :half].zero_() rows[:, half:].fill_(float("-inf")) + def clear_unaccepted_c128_draft_states( + self, + req_pool_indices: torch.Tensor, + seq_lens: torch.Tensor, + accept_lens: torch.Tensor, + num_draft_tokens: int, + ) -> None: + """Clear offline C128 ring slots written for rejected speculative tokens.""" + if ONLINE_C128 or num_draft_tokens <= 1 or req_pool_indices.numel() == 0: + return + + device = req_pool_indices.device + seq_lens = seq_lens.to(device=device, dtype=torch.long) + accept_lens = accept_lens.to(device=device, dtype=torch.long) + offsets = torch.arange(num_draft_tokens, device=device, dtype=torch.long) + reject_mask = offsets.unsqueeze(0) >= accept_lens.unsqueeze(1) + + req_pool_indices = req_pool_indices.to(device=device, dtype=torch.long) + for pool in self.compress_state_pools: + if pool is None or pool.ratio != 128: + continue + + state = pool.kv_score_buffer.kv_score + slots = (seq_lens.unsqueeze(1) + offsets.unsqueeze(0)) % pool.ring_size + rows = req_pool_indices.unsqueeze(1) * pool.ring_size + slots + rows = rows[reject_mask] + + half = state.shape[-1] // 2 + state[rows, :half].zero_() + state[rows, half:].fill_(float("-inf")) + def get_indexer_compress_states(self, layer_id: int) -> CompressStatePool: self.wait_layer_transfer(layer_id) indexer_compress_state_pool = self.indexer_compress_state_pools[layer_id] diff --git a/python/sglang/srt/speculative/eagle_worker_v2.py b/python/sglang/srt/speculative/eagle_worker_v2.py index 86af77e7d694..3edd9852e8a9 100644 --- a/python/sglang/srt/speculative/eagle_worker_v2.py +++ b/python/sglang/srt/speculative/eagle_worker_v2.py @@ -1496,6 +1496,18 @@ def verify(self, batch: ScheduleBatch): accept_index, ) = eagle_sample(verify_input, batch, logits_output, vocab_mask) new_seq_lens = batch.seq_lens + accept_lens + clear_unaccepted_c128 = getattr( + self.token_to_kv_pool_allocator.get_kvcache(), + "clear_unaccepted_c128_draft_states", + None, + ) + if clear_unaccepted_c128 is not None and not batch.forward_mode.is_idle(): + clear_unaccepted_c128( + batch.req_pool_indices, + batch.seq_lens, + accept_lens, + self.speculative_num_draft_tokens, + ) # Update mamba state for hybrid GDN models after verification commit_mamba_states_after_verify( From fa72841663a164834b5521b7b7a483650af0bc63 Mon Sep 17 00:00:00 2001 From: zhangxiaolei Date: Tue, 23 Jun 2026 11:39:01 +0800 Subject: [PATCH 14/24] Optimize offline C128 draft cleanup (#612) --- python/sglang/jit_kernel/dsv4/__init__.py | 2 + python/sglang/jit_kernel/dsv4/c128_cleanup.py | 58 +++++++++++++++++++ .../srt/mem_cache/deepseek_v4_memory_pool.py | 30 +++++----- 3 files changed, 74 insertions(+), 16 deletions(-) create mode 100644 python/sglang/jit_kernel/dsv4/c128_cleanup.py diff --git a/python/sglang/jit_kernel/dsv4/__init__.py b/python/sglang/jit_kernel/dsv4/__init__.py index e344599b0f7e..6f234ca9a4f0 100644 --- a/python/sglang/jit_kernel/dsv4/__init__.py +++ b/python/sglang/jit_kernel/dsv4/__init__.py @@ -10,6 +10,7 @@ compress_norm_rope_store, ) from .compress_old import fused_norm_rope_inplace +from .c128_cleanup import clear_unaccepted_c128_draft_states from .elementwise import ( fused_k_norm_rope_flashmla, fused_q_indexer_rope_hadamard_fp4_quant, @@ -34,6 +35,7 @@ "CompressorPrefillPlan", "compress_forward", "compress_norm_rope_store", + "clear_unaccepted_c128_draft_states", "fused_norm_rope_inplace", "fused_store_cache", "fused_rope_inplace", diff --git a/python/sglang/jit_kernel/dsv4/c128_cleanup.py b/python/sglang/jit_kernel/dsv4/c128_cleanup.py new file mode 100644 index 000000000000..38b3a0ab283b --- /dev/null +++ b/python/sglang/jit_kernel/dsv4/c128_cleanup.py @@ -0,0 +1,58 @@ +import torch +import triton +import triton.language as tl + + +@triton.jit +def _clear_unaccepted_c128_draft_states_kernel( + state, + req_pool_indices, + seq_lens, + accept_lens, + ring_size: tl.constexpr, + half: tl.constexpr, + num_draft_tokens: tl.constexpr, + BLOCK_D: tl.constexpr, +): + bid = tl.program_id(0) + draft_offset = tl.program_id(1) + block_id = tl.program_id(2) + + accept_len = tl.load(accept_lens + bid) + if draft_offset < accept_len: + return + + req_pool_idx = tl.load(req_pool_indices + bid).to(tl.int64) + seq_len = tl.load(seq_lens + bid).to(tl.int64) + slot = (seq_len + draft_offset) % ring_size + row = req_pool_idx * ring_size + slot + + offsets = block_id * BLOCK_D + tl.arange(0, BLOCK_D) + mask = offsets < half + row_base = row * (half * 2) + tl.store(state + row_base + offsets, 0.0, mask=mask) + tl.store(state + row_base + half + offsets, float("-inf"), mask=mask) + + +def clear_unaccepted_c128_draft_states( + state: torch.Tensor, + req_pool_indices: torch.Tensor, + seq_lens: torch.Tensor, + accept_lens: torch.Tensor, + *, + ring_size: int, + num_draft_tokens: int, +) -> None: + half = state.shape[-1] // 2 + _clear_unaccepted_c128_draft_states_kernel[ + (req_pool_indices.numel(), num_draft_tokens, triton.cdiv(half, 256)) + ]( + state, + req_pool_indices, + seq_lens, + accept_lens, + ring_size, + half, + num_draft_tokens, + BLOCK_D=256, + ) diff --git a/python/sglang/srt/mem_cache/deepseek_v4_memory_pool.py b/python/sglang/srt/mem_cache/deepseek_v4_memory_pool.py index d1b298492950..e0055638aa76 100644 --- a/python/sglang/srt/mem_cache/deepseek_v4_memory_pool.py +++ b/python/sglang/srt/mem_cache/deepseek_v4_memory_pool.py @@ -6,7 +6,11 @@ import torch -from sglang.jit_kernel.dsv4 import fused_k_norm_rope_flashmla, fused_store_cache +from sglang.jit_kernel.dsv4 import ( + clear_unaccepted_c128_draft_states, + fused_k_norm_rope_flashmla, + fused_store_cache, +) from sglang.srt.constants import GPU_MEMORY_TYPE_KV_CACHE from sglang.srt.environ import envs from sglang.srt.layers.attention.dsa import index_buf_accessor @@ -1029,25 +1033,19 @@ def clear_unaccepted_c128_draft_states( if ONLINE_C128 or num_draft_tokens <= 1 or req_pool_indices.numel() == 0: return - device = req_pool_indices.device - seq_lens = seq_lens.to(device=device, dtype=torch.long) - accept_lens = accept_lens.to(device=device, dtype=torch.long) - offsets = torch.arange(num_draft_tokens, device=device, dtype=torch.long) - reject_mask = offsets.unsqueeze(0) >= accept_lens.unsqueeze(1) - - req_pool_indices = req_pool_indices.to(device=device, dtype=torch.long) + bs = req_pool_indices.numel() for pool in self.compress_state_pools: if pool is None or pool.ratio != 128: continue - state = pool.kv_score_buffer.kv_score - slots = (seq_lens.unsqueeze(1) + offsets.unsqueeze(0)) % pool.ring_size - rows = req_pool_indices.unsqueeze(1) * pool.ring_size + slots - rows = rows[reject_mask] - - half = state.shape[-1] // 2 - state[rows, :half].zero_() - state[rows, half:].fill_(float("-inf")) + clear_unaccepted_c128_draft_states( + pool.kv_score_buffer.kv_score, + req_pool_indices, + seq_lens, + accept_lens, + ring_size=pool.ring_size, + num_draft_tokens=num_draft_tokens, + ) def get_indexer_compress_states(self, layer_id: int) -> CompressStatePool: self.wait_layer_transfer(layer_id) From 6cf3088af86670432503d6694b3f944085b646d4 Mon Sep 17 00:00:00 2001 From: zhangxiaolei Date: Tue, 23 Jun 2026 21:41:29 +0800 Subject: [PATCH 15/24] Use logical input length for C128 PD state (#613) --- python/sglang/jit_kernel/dsv4/__init__.py | 2 +- python/sglang/srt/disaggregation/base/conn.py | 4 --- .../sglang/srt/disaggregation/common/conn.py | 7 ---- python/sglang/srt/disaggregation/decode.py | 4 +-- python/sglang/srt/disaggregation/fake/conn.py | 1 - .../srt/disaggregation/mooncake/conn.py | 28 +++------------- python/sglang/srt/disaggregation/mori/conn.py | 33 ++++--------------- python/sglang/srt/disaggregation/nixl/conn.py | 23 +++---------- python/sglang/srt/disaggregation/prefill.py | 26 ++++++--------- python/sglang/srt/disaggregation/utils.py | 8 ++++- .../mem_cache/test_dsv4_c128_radix_state.py | 4 +-- 11 files changed, 38 insertions(+), 102 deletions(-) diff --git a/python/sglang/jit_kernel/dsv4/__init__.py b/python/sglang/jit_kernel/dsv4/__init__.py index 6f234ca9a4f0..68cc0e681e0e 100644 --- a/python/sglang/jit_kernel/dsv4/__init__.py +++ b/python/sglang/jit_kernel/dsv4/__init__.py @@ -3,6 +3,7 @@ get_paged_mqa_logits_metadata, triton_create_paged_compress_data, ) +from .c128_cleanup import clear_unaccepted_c128_draft_states from .compress import ( CompressorDecodePlan, CompressorPrefillPlan, @@ -10,7 +11,6 @@ compress_norm_rope_store, ) from .compress_old import fused_norm_rope_inplace -from .c128_cleanup import clear_unaccepted_c128_draft_states from .elementwise import ( fused_k_norm_rope_flashmla, fused_q_indexer_rope_hadamard_fp4_quant, diff --git a/python/sglang/srt/disaggregation/base/conn.py b/python/sglang/srt/disaggregation/base/conn.py index 67505e955479..e998e16bcceb 100644 --- a/python/sglang/srt/disaggregation/base/conn.py +++ b/python/sglang/srt/disaggregation/base/conn.py @@ -134,9 +134,6 @@ def send( def pop_decode_prefix_len(self) -> int: return 0 - def pop_transfer_input_len(self) -> Optional[int]: - return None - def should_send_kv_chunk(self, num_pages: int, last_chunk: bool) -> bool: return num_pages > 0 @@ -187,7 +184,6 @@ def send_metadata( aux_index: Optional[int] = None, state_indices: Optional[List] = None, decode_prefix_len: Optional[int] = None, - transfer_input_len: Optional[int] = None, ): """ Notify the prefill server about the kv indices, aux index, and state_indices. diff --git a/python/sglang/srt/disaggregation/common/conn.py b/python/sglang/srt/disaggregation/common/conn.py index e7b54cccfa7b..33de94719e4d 100644 --- a/python/sglang/srt/disaggregation/common/conn.py +++ b/python/sglang/srt/disaggregation/common/conn.py @@ -175,7 +175,6 @@ def __init__( self.register_to_bootstrap() self.transfer_infos = {} self.req_to_decode_prefix_len: Dict[int, int] = {} - self.req_to_transfer_input_len: Dict[int, int] = {} self.decode_kv_args_table = {} self.pp_group = get_pp_group() # If a timeout happens on the prefill side, it means prefill instances @@ -839,9 +838,6 @@ def init(self, num_kv_indices: int, aux_index: Optional[int] = None): def pop_decode_prefix_len(self) -> int: return self.kv_mgr.req_to_decode_prefix_len.pop(self.bootstrap_room, 0) - def pop_transfer_input_len(self) -> Optional[int]: - return self.kv_mgr.req_to_transfer_input_len.pop(self.bootstrap_room, None) - def should_send_kv_chunk(self, num_pages: int, last_chunk: bool) -> bool: return num_pages > 0 or last_chunk @@ -930,8 +926,6 @@ def clear(self) -> None: self.kv_mgr.request_status.pop(self.bootstrap_room, None) if hasattr(self.kv_mgr, "req_to_decode_prefix_len"): self.kv_mgr.req_to_decode_prefix_len.pop(self.bootstrap_room, None) - if hasattr(self.kv_mgr, "req_to_transfer_input_len"): - self.kv_mgr.req_to_transfer_input_len.pop(self.bootstrap_room, None) if hasattr(self.kv_mgr, "transfer_infos"): self.kv_mgr.transfer_infos.pop(self.bootstrap_room, None) @@ -1143,7 +1137,6 @@ def send_metadata( aux_index: Optional[int] = None, state_indices: Optional[List[int]] = None, decode_prefix_len: Optional[int] = None, - transfer_input_len: Optional[int] = None, ): raise NotImplementedError diff --git a/python/sglang/srt/disaggregation/decode.py b/python/sglang/srt/disaggregation/decode.py index 5ae0ef11a65e..d049039631ae 100644 --- a/python/sglang/srt/disaggregation/decode.py +++ b/python/sglang/srt/disaggregation/decode.py @@ -52,6 +52,7 @@ _is_fake_transfer, get_dsv4_c128_state_indices, get_kv_class, + is_dsv4_c128_online_enabled, is_mla_backend, poll_and_all_reduce, poll_and_all_reduce_with_staging, @@ -1042,7 +1043,7 @@ def _swa_ring_payload(): return ring_rows.astype(np.int32) def _c128_state_payload(): - online = envs.SGLANG_OPT_USE_ONLINE_COMPRESS.get() + online = is_dsv4_c128_online_enabled() ring_size = 1 if online else self.token_to_kv_pool.get_ring_size(128) return get_dsv4_c128_state_indices( int(decode_req.req.req_pool_idx), @@ -1083,7 +1084,6 @@ def _c128_state_payload(): decode_req.metadata_buffer_index, state_indices, decode_prefix_len=total_prefix_len, - transfer_input_len=seq_len, ) if ( self.transfer_queue.enable_staging diff --git a/python/sglang/srt/disaggregation/fake/conn.py b/python/sglang/srt/disaggregation/fake/conn.py index 02373f192573..09c5f80ce492 100644 --- a/python/sglang/srt/disaggregation/fake/conn.py +++ b/python/sglang/srt/disaggregation/fake/conn.py @@ -125,7 +125,6 @@ def send_metadata( aux_index: Optional[int] = None, state_indices: Optional[List] = None, decode_prefix_len: Optional[int] = None, - transfer_input_len: Optional[int] = None, ): self.has_sent_metadata = True logger.debug( diff --git a/python/sglang/srt/disaggregation/mooncake/conn.py b/python/sglang/srt/disaggregation/mooncake/conn.py index ccada456639b..f14c7c9479fc 100644 --- a/python/sglang/srt/disaggregation/mooncake/conn.py +++ b/python/sglang/srt/disaggregation/mooncake/conn.py @@ -76,7 +76,6 @@ class TransferInfo: required_dst_info_num: int is_dummy: bool decode_prefix_len: Optional[int] = None - transfer_input_len: Optional[int] = None # Note: always put the optional staging field at the final (it will be set through 'STAGING_RSP' pkg when needed) staging: Optional[StagingTransferInfo] = None @@ -105,9 +104,6 @@ def from_zmq(cls, msg: List[bytes]): decode_prefix_len=( int(msg[8].decode("ascii")) if len(msg) > 8 and msg[8] != b"" else None ), - transfer_input_len=( - int(msg[9].decode("ascii")) if len(msg) > 9 and msg[9] != b"" else None - ), ) @@ -1018,12 +1014,11 @@ def maybe_send_extra( ) src_indices = list(indices) dst_indices_local = list(dst_indices) - if st == StateType.C128_STATE and len(dst_indices_local) == 0: - # Decode decides whether a request has a partial C128 state - # to receive. At 128-aligned boundaries it intentionally - # registers no C128 dst row and clears the request-scoped - # state locally; a stale prefill-side source row must not - # keep the transfer thread alive or corrupt the next request. + if ( + st == StateType.C128_STATE + and len(src_indices) == 0 + and len(dst_indices_local) == 0 + ): continue if len(src_indices) != len(dst_indices_local): # These components are position- or request-indexed: @@ -1402,7 +1397,6 @@ def transfer_worker( if kv_chunk.room in self.transfer_infos: self.transfer_infos.pop(kv_chunk.room) self.req_to_decode_prefix_len.pop(kv_chunk.room, None) - self.req_to_transfer_input_len.pop(kv_chunk.room, None) except Exception as e: # NOTE(shangming): Remove this when we make sure the transfer thread is bug-free @@ -1504,16 +1498,6 @@ def bootstrap_thread(): ), 0, ) - transfer_input_len = next( - ( - info.transfer_input_len - for info in self.transfer_infos[room].values() - if info.transfer_input_len is not None - ), - None, - ) - if transfer_input_len is not None: - self.req_to_transfer_input_len[room] = transfer_input_len self.update_status(room, KVPoll.WaitingForInput) threading.Thread(target=bootstrap_thread).start() @@ -1859,7 +1843,6 @@ def send_metadata( aux_index: Optional[int] = None, state_indices: Optional[List] = None, decode_prefix_len: Optional[int] = None, - transfer_input_len: Optional[int] = None, ): if self.bootstrap_infos is None: self.kv_mgr.record_failure( @@ -1898,7 +1881,6 @@ def send_metadata( ), str(self.required_dst_info_num).encode("ascii"), str(decode_prefix_len or 0).encode("ascii"), - str(transfer_input_len or 0).encode("ascii"), ] ) self.init_time = time.time() diff --git a/python/sglang/srt/disaggregation/mori/conn.py b/python/sglang/srt/disaggregation/mori/conn.py index da1bdc5d61a5..ef2d16a8e742 100644 --- a/python/sglang/srt/disaggregation/mori/conn.py +++ b/python/sglang/srt/disaggregation/mori/conn.py @@ -124,7 +124,6 @@ class TransferInfo: # populate this field (older receiver or radix-cache feature off) -> treat # as 0 (no prefix hit, full send) for backward compatibility. decode_prefix_len: Optional[int] = None - transfer_input_len: Optional[int] = None @classmethod def from_zmq(cls, payload: List[bytes]) -> TransferInfo: @@ -157,11 +156,6 @@ def from_zmq(cls, payload: List[bytes]) -> TransferInfo: else: decode_prefix_len = None - if len(payload) > 9 and payload[9]: - transfer_input_len: Optional[int] = int(payload[9].decode("ascii")) - else: - transfer_input_len = None - # A transfer is "dummy" only when the receiver does not need any # kv/aux/state delivered. When decode_prefix_len > 0 and the delta is # exactly zero (full prefix hit), dst_kv_indices is empty but aux is @@ -180,7 +174,6 @@ def from_zmq(cls, payload: List[bytes]) -> TransferInfo: required_dst_info_num=required_dst_info_num, is_dummy=is_dummy, decode_prefix_len=decode_prefix_len, - transfer_input_len=transfer_input_len, ) @@ -528,18 +521,6 @@ def _handle_transfer_message(self, payload: List[bytes]) -> None: self.req_to_decode_prefix_len[transfer_info.room] = ( chosen_prefix_len ) - transfer_input_len = next( - ( - info.transfer_input_len - for info in infos.values() - if info.transfer_input_len is not None - ), - None, - ) - if transfer_input_len is not None: - self.req_to_transfer_input_len[transfer_info.room] = ( - transfer_input_len - ) if chosen_prefix_len > 0: # Surface incremental KV transfer at INFO so it's # visible without bumping the global log level. @@ -1240,6 +1221,13 @@ def _send_swa_dsa_state( ) common_len = min(src_state_indices.size, dst_state_indices.size) + if ( + state_type == "c128_state" + and common_len == 0 + and src_state_indices.size == 0 + and dst_state_indices.size == 0 + ): + return [] if common_len == 0 and max(src_state_indices.size, dst_state_indices.size) > 0: raise RuntimeError( f"No overlapping state indices for state_type={state_type}" @@ -1689,7 +1677,6 @@ def send_metadata( aux_index: Optional[int] = None, state_indices: Optional[List] = None, decode_prefix_len: Optional[int] = None, - transfer_input_len: Optional[int] = None, ): if self.bootstrap_infos is None or self.bootstrap_room is None: return @@ -1705,11 +1692,6 @@ def send_metadata( if decode_prefix_len is not None and decode_prefix_len > 0 else b"" ) - transfer_input_len_bytes = ( - str(int(transfer_input_len)).encode("ascii") - if transfer_input_len is not None and transfer_input_len > 0 - else b"" - ) for bootstrap_info in self.bootstrap_infos: sock, lock = self._connect_to_bootstrap_server(bootstrap_info) @@ -1731,7 +1713,6 @@ def send_metadata( state_bytes, str(self.required_dst_info_num).encode("ascii"), decode_prefix_bytes, - transfer_input_len_bytes, ] ) self.init_time = time.time() diff --git a/python/sglang/srt/disaggregation/nixl/conn.py b/python/sglang/srt/disaggregation/nixl/conn.py index 5b9b411efda9..e6b79372e2ef 100644 --- a/python/sglang/srt/disaggregation/nixl/conn.py +++ b/python/sglang/srt/disaggregation/nixl/conn.py @@ -69,7 +69,6 @@ class TransferInfo: required_dst_info_num: int dst_state_indices: List[List[int]] decode_prefix_len: Optional[int] = None # for decode radix cache - transfer_input_len: Optional[int] = None # NOTE: optional staging field; populated via STAGING_RSP. Keep at the # end so positional construction in from_zmq() continues to work. staging: Optional[StagingTransferInfo] = None @@ -101,9 +100,6 @@ def from_zmq(cls, msg: List[bytes]): decode_prefix_len=( int(msg[8].decode("ascii")) if len(msg) > 8 and msg[8] != b"" else None ), # hacky just add it into the message that will be sent - transfer_input_len=( - int(msg[9].decode("ascii")) if len(msg) > 9 and msg[9] != b"" else None - ), ) @@ -874,7 +870,6 @@ def transfer_worker(self, queue: FastQueue, staging_buffer=None): # transfer_worker; staging prefetch sets are NIXL-only). self.transfer_infos.pop(room, None) self.req_to_decode_prefix_len.pop(room, None) - self.req_to_transfer_input_len.pop(room, None) if self.enable_staging and self._staging_ctx is not None: self._staging_ctx.prefetched_rooms.discard(room) self._staging_ctx.prefetch_requested = { @@ -1628,7 +1623,11 @@ def maybe_send_extra( raise RuntimeError( f"PD Disaggregation does NOT support PD different TP sizes for non-MLA {st.upper()} hybrid models yet." ) - if st == StateType.C128_STATE and len(dst_indices) == 0: + if ( + st == StateType.C128_STATE + and len(src_indices) == 0 + and len(dst_indices) == 0 + ): continue if len(src_indices) != len(dst_indices): raise RuntimeError( @@ -1892,16 +1891,6 @@ def bootstrap_thread(): ), 0, ) - transfer_input_len = next( - ( - info.transfer_input_len - for info in self.transfer_infos[room].values() - if info.transfer_input_len is not None - ), - None, - ) - if transfer_input_len is not None: - self.req_to_transfer_input_len[room] = transfer_input_len logger.debug(f"{room=} is bootstrapped") self.update_status(room, KVPoll.WaitingForInput) @@ -2023,7 +2012,6 @@ def send_metadata( aux_index: Optional[int] = None, state_indices: Optional[List] = None, decode_prefix_len: Optional[int] = None, - transfer_input_len: Optional[int] = None, ): if self.bootstrap_infos is None: logger.error( @@ -2071,7 +2059,6 @@ def send_metadata( str(self.required_dst_info_num).encode("ascii"), packed_state_indices, str(decode_prefix_len or 0).encode("ascii"), - str(transfer_input_len or 0).encode("ascii"), ] ) diff --git a/python/sglang/srt/disaggregation/prefill.py b/python/sglang/srt/disaggregation/prefill.py index c9e544e1fd97..540ef1d5ccd2 100644 --- a/python/sglang/srt/disaggregation/prefill.py +++ b/python/sglang/srt/disaggregation/prefill.py @@ -42,6 +42,7 @@ get_dsv4_c128_state_indices, get_kv_class, is_aborted, + is_dsv4_c128_online_enabled, is_mla_backend, poll_and_all_reduce_attn_cp_tp_group, prepare_abort, @@ -272,11 +273,7 @@ def finalize_bootstrap(self, req: Req) -> bool: req.time_stats.set_bootstrap_done_time() decode_prefix_len = req.disagg_kv_sender.pop_decode_prefix_len() - transfer_input_len = req.disagg_kv_sender.pop_transfer_input_len() - if transfer_input_len is None: - transfer_input_len = getattr( - req, "disagg_transfer_input_len", len(req.origin_input_ids) - ) + transfer_input_len = len(req.origin_input_ids) req.disagg_transfer_input_len = transfer_input_len num_kv_indices = transfer_input_len req.start_send_idx = decode_prefix_len @@ -977,9 +974,7 @@ def send_kv_chunk( req, "disagg_transfer_input_len", len(req.origin_input_ids) ) end_idx = ( - end_idx - if end_idx is not None - else min(req.fill_len, transfer_input_len) + end_idx if end_idx is not None else min(req.fill_len, transfer_input_len) ) if not last_chunk: @@ -1004,13 +999,12 @@ def send_kv_chunk( if last_chunk: self.disagg_metadata_buffers.set_buf(req) - # fill_ids includes the token sampled during prefill, but decode - # registers state pages over origin_input_ids (DecodePreallocQueue) - # and the main pool send is clamped to end_idx above. Matching that - # length here avoids emitting an extra state page when the sampled - # token crosses a page boundary, which mismatched src/dst lengths in - # group_concurrent_contiguous. + # Most state payloads read token-pool rows and should match the KV + # range actually materialized on prefill. C128 state is request + # scoped, so its transfer index must use the logical input length + # that decode used to register the destination row. seq_len = min(req.fill_len, transfer_input_len) + c128_seq_len = transfer_input_len def _mamba_payload(): return [ @@ -1057,7 +1051,7 @@ def _swa_ring_payload(): return ring_rows.astype(np.int32) def _c128_state_payload(): - online = envs.SGLANG_OPT_USE_ONLINE_COMPRESS.get() + online = is_dsv4_c128_online_enabled() ring_size = ( 1 if online @@ -1067,7 +1061,7 @@ def _c128_state_payload(): ) return get_dsv4_c128_state_indices( int(req.req_pool_idx), - seq_len, + c128_seq_len, online=online, ring_size=ring_size, ) diff --git a/python/sglang/srt/disaggregation/utils.py b/python/sglang/srt/disaggregation/utils.py index b2315b31ac81..6ea7c4c07652 100644 --- a/python/sglang/srt/disaggregation/utils.py +++ b/python/sglang/srt/disaggregation/utils.py @@ -13,7 +13,7 @@ from sglang.srt.disaggregation.base import KVPoll from sglang.srt.environ import envs -from sglang.srt.utils import is_npu +from sglang.srt.utils import is_hip, is_npu if TYPE_CHECKING: from sglang.srt.disaggregation.base.conn import KVArgs, StateType @@ -30,6 +30,12 @@ # Constants & Enums ######################### FAKE_BOOTSTRAP_HOST = "2.2.2.2" +_IS_HIP = is_hip() + + +def is_dsv4_c128_online_enabled() -> bool: + """Return whether DSV4 C128 uses request-scoped online state.""" + return not _IS_HIP and envs.SGLANG_OPT_USE_ONLINE_COMPRESS.get() def get_dsv4_c128_state_indices( diff --git a/test/registered/unit/mem_cache/test_dsv4_c128_radix_state.py b/test/registered/unit/mem_cache/test_dsv4_c128_radix_state.py index fe54ca703d2d..29cb9e44f56a 100644 --- a/test/registered/unit/mem_cache/test_dsv4_c128_radix_state.py +++ b/test/registered/unit/mem_cache/test_dsv4_c128_radix_state.py @@ -202,9 +202,7 @@ def test_offline_snapshot_only_copies_current_c128_block(self): old_online_c128 = dsv4_memory_pool.ONLINE_C128 dsv4_memory_pool.ONLINE_C128 = False try: - snapshot = kv_pool.snapshot_c128_radix_state(req_pool_idx=1, seq_len=130)[ - 0 - ] + snapshot = kv_pool.snapshot_c128_radix_state(req_pool_idx=1, seq_len=130)[0] finally: dsv4_memory_pool.ONLINE_C128 = old_online_c128 From 3d0dd2429da415cac1dfc24aba5bb714ec2e3a5c Mon Sep 17 00:00:00 2001 From: zhangxiaolei Date: Wed, 24 Jun 2026 10:25:14 +0800 Subject: [PATCH 16/24] Clean up prefill transfer input length (#615) --- python/sglang/srt/disaggregation/prefill.py | 9 ++------- python/sglang/srt/mem_cache/swa_radix_cache.py | 8 +++++++- .../mem_cache/test_dsv4_c128_radix_state.py | 18 ++++++++++++++++++ 3 files changed, 27 insertions(+), 8 deletions(-) diff --git a/python/sglang/srt/disaggregation/prefill.py b/python/sglang/srt/disaggregation/prefill.py index 540ef1d5ccd2..ca0409aaca72 100644 --- a/python/sglang/srt/disaggregation/prefill.py +++ b/python/sglang/srt/disaggregation/prefill.py @@ -249,7 +249,6 @@ def create_sender(self, req: Req, num_kv_heads: int) -> bool: dest_tp_ranks=dest_tp_ranks, pp_rank=self.pp_rank, ) - req.disagg_transfer_input_len = len(req.origin_input_ids) self._process_req(req) req.pending_bootstrap = True return True @@ -273,9 +272,7 @@ def finalize_bootstrap(self, req: Req) -> bool: req.time_stats.set_bootstrap_done_time() decode_prefix_len = req.disagg_kv_sender.pop_decode_prefix_len() - transfer_input_len = len(req.origin_input_ids) - req.disagg_transfer_input_len = transfer_input_len - num_kv_indices = transfer_input_len + num_kv_indices = len(req.origin_input_ids) req.start_send_idx = decode_prefix_len num_kv_indices_to_send = num_kv_indices - decode_prefix_len num_pages = kv_to_page_num( @@ -970,9 +967,7 @@ def send_kv_chunk( """ page_size = self.token_to_kv_pool_allocator.page_size start_idx = req.start_send_idx - transfer_input_len = getattr( - req, "disagg_transfer_input_len", len(req.origin_input_ids) - ) + transfer_input_len = len(req.origin_input_ids) end_idx = ( end_idx if end_idx is not None else min(req.fill_len, transfer_input_len) ) diff --git a/python/sglang/srt/mem_cache/swa_radix_cache.py b/python/sglang/srt/mem_cache/swa_radix_cache.py index d442a1bd01cb..99b3490d0b35 100644 --- a/python/sglang/srt/mem_cache/swa_radix_cache.py +++ b/python/sglang/srt/mem_cache/swa_radix_cache.py @@ -489,7 +489,13 @@ def restore_c128_state_for_reqs(self, reqs: List[Req]) -> None: snapshot = self._get_c128_snapshot_at_node_end(node) if node is self.root_node or not self._is_c128_restorable_node(node): continue - assert self._node_prefix_len(node) == prefix_len + node_prefix_len = self._node_prefix_len(node) + if node_prefix_len < prefix_len: + # Chunked prefill can keep live, non-radix KV from the previous + # chunk in prefix_indices. The request slot already has the + # matching C128 state, so radix restore must leave it intact. + continue + assert node_prefix_len == prefix_len if snapshot is not None: kv_pool.restore_c128_radix_state(int(req.req_pool_idx), snapshot) else: diff --git a/test/registered/unit/mem_cache/test_dsv4_c128_radix_state.py b/test/registered/unit/mem_cache/test_dsv4_c128_radix_state.py index 29cb9e44f56a..7795572cde2a 100644 --- a/test/registered/unit/mem_cache/test_dsv4_c128_radix_state.py +++ b/test/registered/unit/mem_cache/test_dsv4_c128_radix_state.py @@ -172,6 +172,24 @@ def test_missing_partial_snapshot_falls_back_to_aligned_boundary(self): self.assertEqual(kv_pool.restored, []) self.assertEqual(kv_pool.cleared, [3]) + def test_chunked_prefill_live_tail_skips_restore(self): + cache, kv_pool = self._make_cache() + self._insert_tokens(cache, 260) + + req = FakeReq(req_pool_idx=3) + match = cache.match_prefix( + MatchPrefixParams(key=RadixKey(array("q", range(128))), req=req) + ) + + req.prefix_indices = torch.cat( + [match.device_indices, torch.arange(128, 160, dtype=torch.int64)] + ) + req.last_node = match.last_device_node + cache.restore_c128_state_for_reqs([req]) + + self.assertEqual(kv_pool.restored, []) + self.assertEqual(kv_pool.cleared, []) + def test_store_partial_snapshot_uses_requested_prefix_length(self): cache, kv_pool = self._make_cache() leaf = self._insert_tokens(cache, 260) From 18c78cf7a9de1ada37c1536d3130833140c89063 Mon Sep 17 00:00:00 2001 From: zhangxiaolei Date: Wed, 24 Jun 2026 11:16:59 +0800 Subject: [PATCH 17/24] Tighten C128 radix restore live-tail guard (#616) --- python/sglang/srt/mem_cache/swa_radix_cache.py | 3 ++- test/registered/unit/mem_cache/test_dsv4_c128_radix_state.py | 2 ++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/python/sglang/srt/mem_cache/swa_radix_cache.py b/python/sglang/srt/mem_cache/swa_radix_cache.py index 99b3490d0b35..78fd766987a7 100644 --- a/python/sglang/srt/mem_cache/swa_radix_cache.py +++ b/python/sglang/srt/mem_cache/swa_radix_cache.py @@ -494,7 +494,8 @@ def restore_c128_state_for_reqs(self, reqs: List[Req]) -> None: # Chunked prefill can keep live, non-radix KV from the previous # chunk in prefix_indices. The request slot already has the # matching C128 state, so radix restore must leave it intact. - continue + if getattr(req, "kv_committed_len", 0) >= prefix_len: + continue assert node_prefix_len == prefix_len if snapshot is not None: kv_pool.restore_c128_radix_state(int(req.req_pool_idx), snapshot) diff --git a/test/registered/unit/mem_cache/test_dsv4_c128_radix_state.py b/test/registered/unit/mem_cache/test_dsv4_c128_radix_state.py index 7795572cde2a..7c007df7e310 100644 --- a/test/registered/unit/mem_cache/test_dsv4_c128_radix_state.py +++ b/test/registered/unit/mem_cache/test_dsv4_c128_radix_state.py @@ -60,6 +60,7 @@ def __init__(self, req_pool_idx: int): self.req_pool_idx = req_pool_idx self.prefix_indices = [] self.last_node = None + self.kv_committed_len = 0 class FakeKVScoreBuffer: @@ -184,6 +185,7 @@ def test_chunked_prefill_live_tail_skips_restore(self): req.prefix_indices = torch.cat( [match.device_indices, torch.arange(128, 160, dtype=torch.int64)] ) + req.kv_committed_len = len(req.prefix_indices) req.last_node = match.last_device_node cache.restore_c128_state_for_reqs([req]) From ed3e5bc3ba51b55eda0238cd9d53528b4c2f8b55 Mon Sep 17 00:00:00 2001 From: zhangxiaolei Date: Thu, 25 Jun 2026 10:18:22 +0800 Subject: [PATCH 18/24] Reset C128 state on radix cache hits (#617) --- python/sglang/srt/mem_cache/common.py | 6 +- .../srt/mem_cache/deepseek_v4_memory_pool.py | 59 +------- .../sglang/srt/mem_cache/swa_radix_cache.py | 123 +--------------- .../mem_cache/test_dsv4_c128_radix_state.py | 134 ++++-------------- 4 files changed, 39 insertions(+), 283 deletions(-) diff --git a/python/sglang/srt/mem_cache/common.py b/python/sglang/srt/mem_cache/common.py index fc65c8d83866..70d8b72128bc 100644 --- a/python/sglang/srt/mem_cache/common.py +++ b/python/sglang/srt/mem_cache/common.py @@ -505,9 +505,9 @@ def alloc_for_extend( batch.req_to_token_pool, ) - restore_c128_state = getattr(batch.tree_cache, "restore_c128_state_for_reqs", None) - if restore_c128_state is not None: - restore_c128_state(batch.reqs) + reset_c128_state = getattr(batch.tree_cache, "reset_c128_state_for_reqs", None) + if reset_c128_state is not None: + reset_c128_state(batch.reqs) # DSV4-NPU hook: write c4/c128/swa per-req tables from the stashed bundle. # No-op on non-DSV4 paths (out_cache_loc_dsv4 stays None there). diff --git a/python/sglang/srt/mem_cache/deepseek_v4_memory_pool.py b/python/sglang/srt/mem_cache/deepseek_v4_memory_pool.py index e0055638aa76..f02622f7cc39 100644 --- a/python/sglang/srt/mem_cache/deepseek_v4_memory_pool.py +++ b/python/sglang/srt/mem_cache/deepseek_v4_memory_pool.py @@ -946,65 +946,8 @@ def get_online_c128_mtp_pending_seq_lens(self) -> torch.Tensor: assert self.online_c128_mtp_pending_seq_lens is not None return self.online_c128_mtp_pending_seq_lens - def snapshot_c128_radix_state( - self, req_pool_idx: int, seq_len: int - ) -> Optional[List[torch.Tensor]]: - """Snapshot request-scoped C128 state for a radix cache node.""" - if seq_len % 128 == 0: - return None - - snapshots: List[torch.Tensor] = [] - for pool in self.compress_state_pools: - if pool is None or pool.ratio != 128: - continue - state = pool.kv_score_buffer.kv_score - if ONLINE_C128: - snapshots.append(state[req_pool_idx : req_pool_idx + 1].clone()) - else: - start = req_pool_idx * pool.ring_size - state_len = seq_len % 128 - if state_len == 0: - continue - snapshot = state.new_empty((pool.ring_size, state.shape[-1])) - half = snapshot.shape[-1] // 2 - snapshot[:, :half].zero_() - snapshot[:, half:].fill_(float("-inf")) - positions = torch.arange( - seq_len - state_len, - seq_len, - device=state.device, - dtype=torch.long, - ) - slots = positions % pool.ring_size - snapshot[slots] = state[start + slots] - snapshots.append(snapshot) - return snapshots or None - - def restore_c128_radix_state( - self, req_pool_idx: int, snapshots: Optional[List[torch.Tensor]] - ) -> None: - """Restore radix-cached C128 state into the current request slot.""" - if not snapshots: - return - - snapshot_idx = 0 - for pool in self.compress_state_pools: - if pool is None or pool.ratio != 128: - continue - snapshot = snapshots[snapshot_idx] - snapshot_idx += 1 - - state = pool.kv_score_buffer.kv_score - if ONLINE_C128: - state[req_pool_idx : req_pool_idx + 1].copy_(snapshot) - else: - start = req_pool_idx * pool.ring_size - state[start : start + pool.ring_size].copy_(snapshot) - - assert snapshot_idx == len(snapshots) - def clear_c128_radix_state(self, req_pool_idx: int) -> None: - """Reset request-scoped C128 state at a radix-restorable boundary.""" + """Reset request-scoped C128 state at a C128 block boundary.""" for pool in self.compress_state_pools: if pool is None or pool.ratio != 128: continue diff --git a/python/sglang/srt/mem_cache/swa_radix_cache.py b/python/sglang/srt/mem_cache/swa_radix_cache.py index 78fd766987a7..4490662529a7 100644 --- a/python/sglang/srt/mem_cache/swa_radix_cache.py +++ b/python/sglang/srt/mem_cache/swa_radix_cache.py @@ -22,7 +22,7 @@ import heapq import time from collections import defaultdict -from typing import TYPE_CHECKING, Dict, List, Optional, Tuple +from typing import TYPE_CHECKING, List, Optional, Tuple import torch from numpy import float64 @@ -80,9 +80,6 @@ def __init__(self, id: Optional[int] = None): self.host_value = None # store hash values of each page self.hash_value: Optional[List[str]] = None - # C128 state snapshots keyed by the local token offset inside this node. - self.c128_state_snapshots: Optional[Dict[int, List[torch.Tensor]]] = None - # for lru list, invariant: # 1. prev has greater last_access_time # 2. next has smaller last_access_time @@ -395,22 +392,10 @@ def _dsv4_kv_pool(self): if get_kvcache is None: return None kv_pool = get_kvcache() - if ( - not hasattr(kv_pool, "snapshot_c128_radix_state") - or not hasattr(kv_pool, "restore_c128_radix_state") - or not hasattr(kv_pool, "clear_c128_radix_state") - ): + if not hasattr(kv_pool, "clear_c128_radix_state"): return None return kv_pool - def _snapshot_c128_state_for_req( - self, req: Req, seq_len: int - ) -> Optional[List[torch.Tensor]]: - kv_pool = self._dsv4_kv_pool() - if kv_pool is None or req.req_pool_idx is None or seq_len <= 0: - return None - return kv_pool.snapshot_c128_radix_state(int(req.req_pool_idx), seq_len) - def _node_prefix_len(self, node: TreeNode) -> int: prefix_len = 0 while node is not None and node is not self.root_node: @@ -418,66 +403,7 @@ def _node_prefix_len(self, node: TreeNode) -> int: node = node.parent return prefix_len - def _get_c128_snapshot_at_node_end( - self, node: TreeNode - ) -> Optional[List[torch.Tensor]]: - if node.c128_state_snapshots is None: - return None - return node.c128_state_snapshots.get(len(node.value)) - - def _is_c128_restorable_node(self, node: TreeNode) -> bool: - if node is self.root_node: - return True - if self._get_c128_snapshot_at_node_end(node) is not None: - return True - return self._node_prefix_len(node) % 128 == 0 - - def _store_c128_snapshot_at_prefix_len( - self, - node: TreeNode, - prefix_len: int, - snapshot: Optional[List[torch.Tensor]], - ) -> None: - if snapshot is None or prefix_len <= 0: - return - - path: List[TreeNode] = [] - cur = node - while cur is not None and cur is not self.root_node: - path.append(cur) - cur = cur.parent - - remaining = prefix_len - for cur in reversed(path): - if remaining <= len(cur.value): - if cur.c128_state_snapshots is None: - cur.c128_state_snapshots = {} - cur.c128_state_snapshots[remaining] = snapshot - return - remaining -= len(cur.value) - - def _store_c128_partial_snapshot_for_req( - self, req: Req, node: TreeNode, prefix_len: int - ) -> None: - if prefix_len % 128 == 0: - return - snapshot = self._snapshot_c128_state_for_req(req, prefix_len) - self._store_c128_snapshot_at_prefix_len(node, prefix_len, snapshot) - - def _find_c128_restorable_node(self, node: TreeNode) -> TreeNode: - if self._dsv4_kv_pool() is None: - return node - while node is not self.root_node and not self._is_c128_restorable_node(node): - prefix_len = self._node_prefix_len(node) - aligned_prefix_len = prefix_len // 128 * 128 - parent_prefix_len = prefix_len - len(node.value) - if aligned_prefix_len > parent_prefix_len: - split_len = aligned_prefix_len - parent_prefix_len - return self._split_node(node.key, node, split_len) - node = node.parent - return node - - def restore_c128_state_for_reqs(self, reqs: List[Req]) -> None: + def reset_c128_state_for_reqs(self, reqs: List[Req]) -> None: kv_pool = self._dsv4_kv_pool() if kv_pool is None: return @@ -486,21 +412,18 @@ def restore_c128_state_for_reqs(self, reqs: List[Req]) -> None: continue node = getattr(req, "last_node", self.root_node) prefix_len = len(getattr(req, "prefix_indices", [])) - snapshot = self._get_c128_snapshot_at_node_end(node) - if node is self.root_node or not self._is_c128_restorable_node(node): + if node is self.root_node or prefix_len == 0: continue node_prefix_len = self._node_prefix_len(node) if node_prefix_len < prefix_len: # Chunked prefill can keep live, non-radix KV from the previous # chunk in prefix_indices. The request slot already has the - # matching C128 state, so radix restore must leave it intact. + # matching C128 state, so radix-hit hygiene must leave it intact. if getattr(req, "kv_committed_len", 0) >= prefix_len: continue assert node_prefix_len == prefix_len - if snapshot is not None: - kv_pool.restore_c128_radix_state(int(req.req_pool_idx), snapshot) - else: - kv_pool.clear_c128_radix_state(int(req.req_pool_idx)) + assert prefix_len % 128 == 0 + kv_pool.clear_c128_radix_state(int(req.req_pool_idx)) def match_prefix(self, params: MatchPrefixParams) -> MatchResult: """Find the matching prefix from the radix tree. @@ -584,12 +507,6 @@ def cache_finished_req(self, req: Req, is_insert: bool = True) -> None: swa_evicted_seqlen=req.swa_evicted_seqlen, ) ) - match_result = self.match_prefix(MatchPrefixParams(key=radix_key)) - if len(match_result.device_indices) == page_aligned_len: - last_node = match_result.last_device_node - self._store_c128_partial_snapshot_for_req( - req, last_node, page_aligned_len - ) else: self.token_to_kv_pool_allocator.free( kv_indices[old_prefix_len:page_aligned_len] @@ -673,7 +590,6 @@ def cache_unfinished_req(self, req: Req, chunked=False) -> None: req.prefix_indices = new_indices req.last_node = new_last_node req.swa_uuid_for_lock = swa_uuid_for_lock - self._store_c128_partial_snapshot_for_req(req, new_last_node, len(new_indices)) def pretty_print(self) -> None: self._print_helper(self.root_node, 0) @@ -1083,12 +999,6 @@ def _match_post_processor( else: value = torch.empty((0,), dtype=torch.int64, device=self.device) - if params.req is not None and len(value) > 0: - last_node = self._find_c128_restorable_node(last_node) - c128_prefix_len = self._node_prefix_len(last_node) - if c128_prefix_len < len(value): - value = value[:c128_prefix_len] - return MatchResult( device_indices=value, last_device_node=last_node, @@ -1126,12 +1036,6 @@ def _compact_single_child_chain(self, node: TreeNode) -> None: is_bigram=node.key.is_bigram, ) node.value = torch.cat([node.value, child.value]) - if child.c128_state_snapshots is not None: - offset = len(node.value) - len(child.value) - if node.c128_state_snapshots is None: - node.c128_state_snapshots = {} - for rel_len, snapshot in child.c128_state_snapshots.items(): - node.c128_state_snapshots[offset + rel_len] = snapshot node.children = child.children for grandchild in node.children.values(): grandchild.parent = node @@ -1210,19 +1114,6 @@ def _split_node(self, key: RadixKey, child: TreeNode, split_len: int) -> TreeNod child.key = child.key[split_len:] assert len(child.key) > 0, f"child.key should not be empty" child.value = child.value[split_len:].clone() - if child.c128_state_snapshots is not None: - parent_snapshots = { - rel_len: snapshot - for rel_len, snapshot in child.c128_state_snapshots.items() - if rel_len <= split_len - } - child_snapshots = { - rel_len - split_len: snapshot - for rel_len, snapshot in child.c128_state_snapshots.items() - if rel_len > split_len - } - new_node.c128_state_snapshots = parent_snapshots or None - child.c128_state_snapshots = child_snapshots or None new_node.parent.children[key.child_key(self.page_size)] = new_node new_node.hash_value, child.hash_value = split_node_hash_value( child.hash_value, split_len, self.page_size diff --git a/test/registered/unit/mem_cache/test_dsv4_c128_radix_state.py b/test/registered/unit/mem_cache/test_dsv4_c128_radix_state.py index 7c007df7e310..26e087651800 100644 --- a/test/registered/unit/mem_cache/test_dsv4_c128_radix_state.py +++ b/test/registered/unit/mem_cache/test_dsv4_c128_radix_state.py @@ -3,7 +3,6 @@ import torch -from sglang.srt.mem_cache import deepseek_v4_memory_pool as dsv4_memory_pool from sglang.srt.mem_cache.allocator.swa import SWATokenToKVPoolAllocator from sglang.srt.mem_cache.base_prefix_cache import InsertParams, MatchPrefixParams from sglang.srt.mem_cache.base_swa_memory_pool import BaseSWAKVPool @@ -19,9 +18,7 @@ class FakeDSV4KVPool(BaseSWAKVPool): def __init__(self): self.full_kv_pool = None self.swa_kv_pool = None - self.restored = [] self.cleared = [] - self.snapshotted = [] def get_key_buffer(self, layer_id: int) -> torch.Tensor: raise NotImplementedError @@ -44,13 +41,6 @@ def translate_loc_from_full_to_swa(self, kv_indices: torch.Tensor) -> torch.Tens def get_state_buf_infos(self): return [], [], [] - def snapshot_c128_radix_state(self, req_pool_idx: int, seq_len: int): - self.snapshotted.append((req_pool_idx, seq_len)) - return [torch.tensor([req_pool_idx, seq_len], dtype=torch.int64)] - - def restore_c128_radix_state(self, req_pool_idx: int, snapshots): - self.restored.append((req_pool_idx, snapshots)) - def clear_c128_radix_state(self, req_pool_idx: int): self.cleared.append(req_pool_idx) @@ -63,26 +53,13 @@ def __init__(self, req_pool_idx: int): self.kv_committed_len = 0 -class FakeKVScoreBuffer: - def __init__(self, kv_score: torch.Tensor): - self.kv_score = kv_score - - -class FakeCompressStatePool: - ratio = 128 - - def __init__(self, ring_size: int, kv_score: torch.Tensor): - self.ring_size = ring_size - self.kv_score_buffer = FakeKVScoreBuffer(kv_score) - - class TestDSV4C128RadixState(unittest.TestCase): def _make_cache(self): kv_pool = FakeDSV4KVPool() allocator = SWATokenToKVPoolAllocator( size=512, size_swa=512, - page_size=1, + page_size=256, dtype=torch.float16, device="cpu", kvcache=kv_pool, @@ -96,7 +73,7 @@ def _make_cache(self): disable=False, req_to_token_pool=req_to_token_pool, token_to_kv_pool_allocator=allocator, - page_size=1, + page_size=256, sliding_window_size=512, ) ) @@ -107,131 +84,76 @@ def _insert_tokens(self, cache: SWARadixCache, length: int): value = torch.arange(length, dtype=torch.int64) cache.insert(InsertParams(key=key, value=value)) match = cache.match_prefix(MatchPrefixParams(key=key)) - self.assertEqual(len(match.device_indices), length) + self.assertEqual(len(match.device_indices), length // 256 * 256) return match.last_device_node - def test_aligned_c128_boundary_does_not_need_snapshot(self): + def test_page_aligned_radix_hit_resets_c128_state(self): cache, kv_pool = self._make_cache() self._insert_tokens(cache, 260) req = FakeReq(req_pool_idx=2) match = cache.match_prefix( - MatchPrefixParams(key=RadixKey(array("q", range(128))), req=req) + MatchPrefixParams(key=RadixKey(array("q", range(256))), req=req) ) - self.assertEqual(len(match.device_indices), 128) - self.assertIsNone(match.last_device_node.c128_state_snapshots) + self.assertEqual(len(match.device_indices), 256) req.prefix_indices = match.device_indices req.last_node = match.last_device_node - cache.restore_c128_state_for_reqs([req]) - self.assertEqual(kv_pool.restored, []) + cache.reset_c128_state_for_reqs([req]) self.assertEqual(kv_pool.cleared, [2]) - def test_partial_c128_snapshot_moves_to_split_node_and_restores(self): - cache, kv_pool = self._make_cache() - leaf = self._insert_tokens(cache, 260) - - snapshot = [torch.tensor([7, 130], dtype=torch.int64)] - cache._store_c128_snapshot_at_prefix_len(leaf, 130, snapshot) - - req = FakeReq(req_pool_idx=3) - match = cache.match_prefix( - MatchPrefixParams(key=RadixKey(array("q", range(130))), req=req) - ) - - self.assertEqual(len(match.device_indices), 130) - self.assertIs( - match.last_device_node.c128_state_snapshots[130], - snapshot, - ) - - req.prefix_indices = match.device_indices - req.last_node = match.last_device_node - cache.restore_c128_state_for_reqs([req]) - - self.assertEqual(len(kv_pool.restored), 1) - restored_req_pool_idx, restored_snapshot = kv_pool.restored[0] - self.assertEqual(restored_req_pool_idx, 3) - self.assertIs(restored_snapshot, snapshot) - - def test_missing_partial_snapshot_falls_back_to_aligned_boundary(self): + def test_unaligned_match_uses_aligned_boundary_and_resets(self): cache, kv_pool = self._make_cache() self._insert_tokens(cache, 260) req = FakeReq(req_pool_idx=3) match = cache.match_prefix( - MatchPrefixParams(key=RadixKey(array("q", range(130))), req=req) + MatchPrefixParams(key=RadixKey(array("q", range(260))), req=req) ) - self.assertEqual(len(match.device_indices), 128) - self.assertEqual(cache._node_prefix_len(match.last_device_node), 128) + self.assertEqual(len(match.device_indices), 256) + self.assertEqual(cache._node_prefix_len(match.last_device_node), 256) req.prefix_indices = match.device_indices req.last_node = match.last_device_node - cache.restore_c128_state_for_reqs([req]) - self.assertEqual(kv_pool.restored, []) + cache.reset_c128_state_for_reqs([req]) self.assertEqual(kv_pool.cleared, [3]) - def test_chunked_prefill_live_tail_skips_restore(self): + def test_chunked_prefill_live_tail_skips_reset(self): cache, kv_pool = self._make_cache() self._insert_tokens(cache, 260) req = FakeReq(req_pool_idx=3) match = cache.match_prefix( - MatchPrefixParams(key=RadixKey(array("q", range(128))), req=req) + MatchPrefixParams(key=RadixKey(array("q", range(256))), req=req) ) req.prefix_indices = torch.cat( - [match.device_indices, torch.arange(128, 160, dtype=torch.int64)] + [match.device_indices, torch.arange(256, 288, dtype=torch.int64)] ) req.kv_committed_len = len(req.prefix_indices) req.last_node = match.last_device_node - cache.restore_c128_state_for_reqs([req]) + cache.reset_c128_state_for_reqs([req]) - self.assertEqual(kv_pool.restored, []) self.assertEqual(kv_pool.cleared, []) - def test_store_partial_snapshot_uses_requested_prefix_length(self): - cache, kv_pool = self._make_cache() - leaf = self._insert_tokens(cache, 260) - req = FakeReq(req_pool_idx=4) - - cache._store_c128_partial_snapshot_for_req(req, leaf, 130) - - self.assertEqual(kv_pool.snapshotted, [(4, 130)]) - self.assertIn(130, leaf.c128_state_snapshots) - - def test_store_aligned_boundary_skips_snapshot(self): + def test_non_live_tail_length_mismatch_asserts(self): cache, kv_pool = self._make_cache() - leaf = self._insert_tokens(cache, 260) - req = FakeReq(req_pool_idx=4) - - cache._store_c128_partial_snapshot_for_req(req, leaf, 128) + self._insert_tokens(cache, 260) - self.assertEqual(kv_pool.snapshotted, []) - self.assertIsNone(leaf.c128_state_snapshots) + req = FakeReq(req_pool_idx=3) + match = cache.match_prefix( + MatchPrefixParams(key=RadixKey(array("q", range(256))), req=req) + ) - def test_offline_snapshot_only_copies_current_c128_block(self): - kv_score = torch.arange(512 * 4, dtype=torch.float32).reshape(512, 4) - kv_pool = dsv4_memory_pool.DeepSeekV4TokenToKVPool.__new__( - dsv4_memory_pool.DeepSeekV4TokenToKVPool + req.prefix_indices = torch.cat( + [match.device_indices, torch.arange(256, 288, dtype=torch.int64)] ) - kv_pool.compress_state_pools = [FakeCompressStatePool(256, kv_score)] - - old_online_c128 = dsv4_memory_pool.ONLINE_C128 - dsv4_memory_pool.ONLINE_C128 = False - try: - snapshot = kv_pool.snapshot_c128_radix_state(req_pool_idx=1, seq_len=130)[0] - finally: - dsv4_memory_pool.ONLINE_C128 = old_online_c128 - - start = 256 - expected = kv_score.new_empty((256, 4)) - expected[:, :2].zero_() - expected[:, 2:].fill_(float("-inf")) - expected[128:130] = kv_score[start + 128 : start + 130] - torch.testing.assert_close(snapshot, expected) + req.last_node = match.last_device_node + with self.assertRaises(AssertionError): + cache.reset_c128_state_for_reqs([req]) + self.assertEqual(kv_pool.cleared, []) if __name__ == "__main__": From bc25dee67c45e9d660adfe14b54594ecc721f9a6 Mon Sep 17 00:00:00 2001 From: zhangxiaolei Date: Thu, 25 Jun 2026 15:50:41 +0800 Subject: [PATCH 19/24] Resolve prefill send kv chunk conflict (#619) --- python/sglang/srt/disaggregation/prefill.py | 18 ++---------------- 1 file changed, 2 insertions(+), 16 deletions(-) diff --git a/python/sglang/srt/disaggregation/prefill.py b/python/sglang/srt/disaggregation/prefill.py index 6558fea4e12a..2c65bd26c447 100644 --- a/python/sglang/srt/disaggregation/prefill.py +++ b/python/sglang/srt/disaggregation/prefill.py @@ -970,13 +970,9 @@ def send_kv_chunk( start_idx = req.start_send_idx transfer_input_len = len(req.origin_input_ids) end_idx = ( -<<<<<<< sync-pr-601-to-main - end_idx if end_idx is not None else min(req.fill_len, transfer_input_len) -======= end_idx if end_idx is not None - else min(req.extend_range.end, len(req.origin_input_ids)) ->>>>>>> main + else min(req.extend_range.end, transfer_input_len) ) if not last_chunk: @@ -1001,22 +997,12 @@ def send_kv_chunk( if last_chunk: self.disagg_metadata_buffers.set_buf(req) -<<<<<<< sync-pr-601-to-main # Most state payloads read token-pool rows and should match the KV # range actually materialized on prefill. C128 state is request # scoped, so its transfer index must use the logical input length # that decode used to register the destination row. - seq_len = min(req.fill_len, transfer_input_len) + seq_len = min(req.extend_range.end, transfer_input_len) c128_seq_len = transfer_input_len -======= - # fill_ids includes the token sampled during prefill, but decode - # registers state pages over origin_input_ids (DecodePreallocQueue) - # and the main pool send is clamped to end_idx above. Matching that - # length here avoids emitting an extra state page when the sampled - # token crosses a page boundary, which mismatched src/dst lengths in - # group_concurrent_contiguous. - seq_len = min(req.extend_range.end, len(req.origin_input_ids)) ->>>>>>> main def _mamba_payload(): return [ From 5d4e944dd4a36b685faea891ff882879fa0ca906 Mon Sep 17 00:00:00 2001 From: zhangxiaolei Date: Thu, 25 Jun 2026 18:19:21 +0800 Subject: [PATCH 20/24] Reset C128 state on req slot allocation (#618) --- python/sglang/srt/disaggregation/decode.py | 2 +- python/sglang/srt/mem_cache/common.py | 47 ++++- .../srt/mem_cache/deepseek_v4_memory_pool.py | 32 ++-- .../sglang/srt/mem_cache/swa_radix_cache.py | 40 ----- .../mem_cache/test_dsv4_c128_radix_state.py | 160 ------------------ .../mem_cache/test_dsv4_c128_req_state.py | 76 +++++++++ 6 files changed, 141 insertions(+), 216 deletions(-) delete mode 100644 test/registered/unit/mem_cache/test_dsv4_c128_radix_state.py create mode 100644 test/registered/unit/mem_cache/test_dsv4_c128_req_state.py diff --git a/python/sglang/srt/disaggregation/decode.py b/python/sglang/srt/disaggregation/decode.py index ac4b876d573d..77f45af04b93 100644 --- a/python/sglang/srt/disaggregation/decode.py +++ b/python/sglang/srt/disaggregation/decode.py @@ -1056,7 +1056,7 @@ def _c128_state_payload(): state_indices: Optional[List] = [] if StateType.C128_STATE in state_types: clear_c128_state = getattr( - self.token_to_kv_pool, "clear_c128_radix_state", None + self.token_to_kv_pool, "clear_c128_req_state", None ) if clear_c128_state is not None: clear_c128_state(int(decode_req.req.req_pool_idx)) diff --git a/python/sglang/srt/mem_cache/common.py b/python/sglang/srt/mem_cache/common.py index 8487226dae2d..43020c5409b5 100644 --- a/python/sglang/srt/mem_cache/common.py +++ b/python/sglang/srt/mem_cache/common.py @@ -47,6 +47,47 @@ logger = logging.getLogger(__name__) +def _clear_c128_req_state_for_new_slots( + tree_cache: BasePrefixCache | None, + reqs: list[Req], + already_allocated: list[bool], +) -> None: + # C128_STATE is request-scoped runtime state indexed by req_pool_idx. When a + # req slot is reused for a new request, stale partial C128 state from the + # previous request must be cleared before the new request starts using it. + # This is intentionally tied to req slot allocation instead of radix-cache + # hits: radix/HiCache hits are page-aligned and reuse complete C128 KV, while + # chunked continuation reuses an existing req slot and must keep its live + # partial state. + if tree_cache is None: + return + + allocator = getattr(tree_cache, "token_to_kv_pool_allocator", None) + if allocator is None: + return + + get_kvcache = getattr(allocator, "get_kvcache", None) + if get_kvcache is None: + return + + kv_pool = get_kvcache() + clear_c128_req_states = getattr(kv_pool, "clear_c128_req_states", None) + clear_c128_req_state = getattr(kv_pool, "clear_c128_req_state", None) + if clear_c128_req_states is None and clear_c128_req_state is None: + return + + new_req_pool_indices = [ + int(req.req_pool_idx) + for req, reused in zip(reqs, already_allocated) + if not reused and req.req_pool_idx is not None + ] + if clear_c128_req_states is not None: + clear_c128_req_states(new_req_pool_indices) + else: + for req_pool_idx in new_req_pool_indices: + clear_c128_req_state(req_pool_idx) + + def kv_to_page_indices(kv_indices: np.ndarray, page_size: int): # The page is guaranteed to be full except the last page. if page_size == 1: @@ -412,6 +453,7 @@ def alloc_req_slots( ) -> list[int]: """Allocate request slots from the pool.""" num_reqs = len(reqs) + already_allocated = [req.req_pool_idx is not None for req in reqs] if isinstance(req_to_token_pool, HybridReqToTokenPool): mamba_available_size = req_to_token_pool.mamba_allocator.available_size() if tree_cache.supports_mamba(): @@ -436,6 +478,7 @@ def alloc_req_slots( f"{req_to_token_pool.available_size()=}, " f"{num_reqs=}, " ) + _clear_c128_req_state_for_new_slots(tree_cache, reqs, already_allocated) return req_pool_indices @@ -517,10 +560,6 @@ def alloc_for_extend( batch.req_to_token_pool, ) - reset_c128_state = getattr(batch.tree_cache, "reset_c128_state_for_reqs", None) - if reset_c128_state is not None: - reset_c128_state(batch.reqs) - # DSV4-NPU hook: write c4/c128/swa per-req tables from the stashed bundle. # No-op on non-DSV4 paths (out_cache_loc_dsv4 stays None there). if _is_npu: diff --git a/python/sglang/srt/mem_cache/deepseek_v4_memory_pool.py b/python/sglang/srt/mem_cache/deepseek_v4_memory_pool.py index db04d03b5750..9840f2be74ec 100644 --- a/python/sglang/srt/mem_cache/deepseek_v4_memory_pool.py +++ b/python/sglang/srt/mem_cache/deepseek_v4_memory_pool.py @@ -946,24 +946,34 @@ def get_online_c128_mtp_pending_seq_lens(self) -> torch.Tensor: assert self.online_c128_mtp_pending_seq_lens is not None return self.online_c128_mtp_pending_seq_lens - def clear_c128_radix_state(self, req_pool_idx: int) -> None: - """Reset request-scoped C128 state at a C128 block boundary.""" + def clear_c128_req_states(self, req_pool_indices: List[int]) -> None: + """Reset request-scoped C128 states for newly allocated req slots.""" + if not req_pool_indices: + return + for pool in self.compress_state_pools: if pool is None or pool.ratio != 128: continue state = pool.kv_score_buffer.kv_score + req_indices = torch.tensor( + req_pool_indices, dtype=torch.long, device=state.device + ) if ONLINE_C128: - row = state[req_pool_idx] - head_dim = row.shape[-1] // 3 - row[:head_dim].fill_(float("-inf")) - row[head_dim:].zero_() + head_dim = state.shape[-1] // 3 + state[req_indices, :head_dim] = float("-inf") + state[req_indices, head_dim:] = 0 else: - start = req_pool_idx * pool.ring_size - rows = state[start : start + pool.ring_size] - half = rows.shape[-1] // 2 - rows[:, :half].zero_() - rows[:, half:].fill_(float("-inf")) + offsets = torch.arange(pool.ring_size, device=state.device) + row_indices = ( + req_indices[:, None] * pool.ring_size + offsets[None, :] + ).reshape(-1) + half = state.shape[-1] // 2 + state[row_indices, :half] = 0 + state[row_indices, half:] = float("-inf") + + def clear_c128_req_state(self, req_pool_idx: int) -> None: + self.clear_c128_req_states([req_pool_idx]) def clear_unaccepted_c128_draft_states( self, diff --git a/python/sglang/srt/mem_cache/swa_radix_cache.py b/python/sglang/srt/mem_cache/swa_radix_cache.py index 8ad86e6a05fb..1846be14bf18 100644 --- a/python/sglang/srt/mem_cache/swa_radix_cache.py +++ b/python/sglang/srt/mem_cache/swa_radix_cache.py @@ -385,46 +385,6 @@ def reset(self) -> None: self.swa_lru_list = LRUList(is_swa_list=True) self._record_all_cleared_event() - def _dsv4_kv_pool(self): - if self.token_to_kv_pool_allocator is None: - return None - get_kvcache = getattr(self.token_to_kv_pool_allocator, "get_kvcache", None) - if get_kvcache is None: - return None - kv_pool = get_kvcache() - if not hasattr(kv_pool, "clear_c128_radix_state"): - return None - return kv_pool - - def _node_prefix_len(self, node: TreeNode) -> int: - prefix_len = 0 - while node is not None and node is not self.root_node: - prefix_len += len(node.value) - node = node.parent - return prefix_len - - def reset_c128_state_for_reqs(self, reqs: List[Req]) -> None: - kv_pool = self._dsv4_kv_pool() - if kv_pool is None: - return - for req in reqs: - if req.req_pool_idx is None: - continue - node = getattr(req, "last_node", self.root_node) - prefix_len = len(getattr(req, "prefix_indices", [])) - if node is self.root_node or prefix_len == 0: - continue - node_prefix_len = self._node_prefix_len(node) - if node_prefix_len < prefix_len: - # Chunked prefill can keep live, non-radix KV from the previous - # chunk in prefix_indices. The request slot already has the - # matching C128 state, so radix-hit hygiene must leave it intact. - if getattr(req, "kv_committed_len", 0) >= prefix_len: - continue - assert node_prefix_len == prefix_len - assert prefix_len % 128 == 0 - kv_pool.clear_c128_radix_state(int(req.req_pool_idx)) - def match_prefix(self, params: MatchPrefixParams) -> MatchResult: """Find the matching prefix from the radix tree. Args: diff --git a/test/registered/unit/mem_cache/test_dsv4_c128_radix_state.py b/test/registered/unit/mem_cache/test_dsv4_c128_radix_state.py deleted file mode 100644 index 26e087651800..000000000000 --- a/test/registered/unit/mem_cache/test_dsv4_c128_radix_state.py +++ /dev/null @@ -1,160 +0,0 @@ -import unittest -from array import array - -import torch - -from sglang.srt.mem_cache.allocator.swa import SWATokenToKVPoolAllocator -from sglang.srt.mem_cache.base_prefix_cache import InsertParams, MatchPrefixParams -from sglang.srt.mem_cache.base_swa_memory_pool import BaseSWAKVPool -from sglang.srt.mem_cache.cache_init_params import CacheInitParams -from sglang.srt.mem_cache.memory_pool import ReqToTokenPool -from sglang.srt.mem_cache.swa_radix_cache import RadixKey, SWARadixCache -from sglang.test.ci.ci_register import register_cpu_ci - -register_cpu_ci(est_time=5, suite="base-b-test-cpu") - - -class FakeDSV4KVPool(BaseSWAKVPool): - def __init__(self): - self.full_kv_pool = None - self.swa_kv_pool = None - self.cleared = [] - - def get_key_buffer(self, layer_id: int) -> torch.Tensor: - raise NotImplementedError - - def get_value_buffer(self, layer_id: int) -> torch.Tensor: - raise NotImplementedError - - def get_kv_buffer(self, layer_id: int): - raise NotImplementedError - - def set_kv_buffer(self, layer, loc, cache_k, cache_v) -> None: - raise NotImplementedError - - def register_mapping(self, full_to_swa_index_mapping: torch.Tensor) -> None: - pass - - def translate_loc_from_full_to_swa(self, kv_indices: torch.Tensor) -> torch.Tensor: - return kv_indices - - def get_state_buf_infos(self): - return [], [], [] - - def clear_c128_radix_state(self, req_pool_idx: int): - self.cleared.append(req_pool_idx) - - -class FakeReq: - def __init__(self, req_pool_idx: int): - self.req_pool_idx = req_pool_idx - self.prefix_indices = [] - self.last_node = None - self.kv_committed_len = 0 - - -class TestDSV4C128RadixState(unittest.TestCase): - def _make_cache(self): - kv_pool = FakeDSV4KVPool() - allocator = SWATokenToKVPoolAllocator( - size=512, - size_swa=512, - page_size=256, - dtype=torch.float16, - device="cpu", - kvcache=kv_pool, - need_sort=False, - ) - req_to_token_pool = ReqToTokenPool( - size=4, max_context_len=512, device="cpu", enable_memory_saver=False - ) - cache = SWARadixCache( - CacheInitParams( - disable=False, - req_to_token_pool=req_to_token_pool, - token_to_kv_pool_allocator=allocator, - page_size=256, - sliding_window_size=512, - ) - ) - return cache, kv_pool - - def _insert_tokens(self, cache: SWARadixCache, length: int): - key = RadixKey(array("q", range(length))) - value = torch.arange(length, dtype=torch.int64) - cache.insert(InsertParams(key=key, value=value)) - match = cache.match_prefix(MatchPrefixParams(key=key)) - self.assertEqual(len(match.device_indices), length // 256 * 256) - return match.last_device_node - - def test_page_aligned_radix_hit_resets_c128_state(self): - cache, kv_pool = self._make_cache() - self._insert_tokens(cache, 260) - - req = FakeReq(req_pool_idx=2) - match = cache.match_prefix( - MatchPrefixParams(key=RadixKey(array("q", range(256))), req=req) - ) - - self.assertEqual(len(match.device_indices), 256) - - req.prefix_indices = match.device_indices - req.last_node = match.last_device_node - cache.reset_c128_state_for_reqs([req]) - self.assertEqual(kv_pool.cleared, [2]) - - def test_unaligned_match_uses_aligned_boundary_and_resets(self): - cache, kv_pool = self._make_cache() - self._insert_tokens(cache, 260) - - req = FakeReq(req_pool_idx=3) - match = cache.match_prefix( - MatchPrefixParams(key=RadixKey(array("q", range(260))), req=req) - ) - - self.assertEqual(len(match.device_indices), 256) - self.assertEqual(cache._node_prefix_len(match.last_device_node), 256) - - req.prefix_indices = match.device_indices - req.last_node = match.last_device_node - cache.reset_c128_state_for_reqs([req]) - self.assertEqual(kv_pool.cleared, [3]) - - def test_chunked_prefill_live_tail_skips_reset(self): - cache, kv_pool = self._make_cache() - self._insert_tokens(cache, 260) - - req = FakeReq(req_pool_idx=3) - match = cache.match_prefix( - MatchPrefixParams(key=RadixKey(array("q", range(256))), req=req) - ) - - req.prefix_indices = torch.cat( - [match.device_indices, torch.arange(256, 288, dtype=torch.int64)] - ) - req.kv_committed_len = len(req.prefix_indices) - req.last_node = match.last_device_node - cache.reset_c128_state_for_reqs([req]) - - self.assertEqual(kv_pool.cleared, []) - - def test_non_live_tail_length_mismatch_asserts(self): - cache, kv_pool = self._make_cache() - self._insert_tokens(cache, 260) - - req = FakeReq(req_pool_idx=3) - match = cache.match_prefix( - MatchPrefixParams(key=RadixKey(array("q", range(256))), req=req) - ) - - req.prefix_indices = torch.cat( - [match.device_indices, torch.arange(256, 288, dtype=torch.int64)] - ) - req.last_node = match.last_device_node - with self.assertRaises(AssertionError): - cache.reset_c128_state_for_reqs([req]) - self.assertEqual(kv_pool.cleared, []) - - -if __name__ == "__main__": - unittest.main() diff --git a/test/registered/unit/mem_cache/test_dsv4_c128_req_state.py b/test/registered/unit/mem_cache/test_dsv4_c128_req_state.py new file mode 100644 index 000000000000..322b16eb6064 --- /dev/null +++ b/test/registered/unit/mem_cache/test_dsv4_c128_req_state.py @@ -0,0 +1,76 @@ +import unittest + +from sglang.srt.mem_cache.common import alloc_req_slots +from sglang.srt.mem_cache.memory_pool import ReqToTokenPool +from sglang.test.ci.ci_register import register_cpu_ci + +register_cpu_ci(est_time=5, suite="base-b-test-cpu") + + +class FakeKVPool: + def __init__(self): + self.cleared = [] + + def clear_c128_req_states(self, req_pool_indices): + self.cleared.append(list(req_pool_indices)) + + +class FakeAllocator: + def __init__(self, kv_pool): + self.kv_pool = kv_pool + + def get_kvcache(self): + return self.kv_pool + + +class FakeTreeCache: + def __init__(self, kv_pool): + self.token_to_kv_pool_allocator = FakeAllocator(kv_pool) + + +class FakeReq: + def __init__(self, req_pool_idx=None): + self.req_pool_idx = req_pool_idx + self.inflight_middle_chunks = 0 + self.kv_committed_len = 0 + + +class TestDSV4C128ReqState(unittest.TestCase): + def _make_pool(self): + return ReqToTokenPool( + size=4, + max_context_len=16, + device="cpu", + enable_memory_saver=False, + ) + + def test_new_req_slots_clear_c128_state(self): + req_to_token_pool = self._make_pool() + kv_pool = FakeKVPool() + reqs = [FakeReq(), FakeReq()] + + req_pool_indices = alloc_req_slots( + req_to_token_pool, reqs, FakeTreeCache(kv_pool) + ) + + self.assertEqual(req_pool_indices, [1, 2]) + self.assertEqual(kv_pool.cleared, [[1, 2]]) + + def test_reused_req_slot_skips_c128_state_clear(self): + req_to_token_pool = self._make_pool() + req_to_token_pool.free_slots.remove(3) + kv_pool = FakeKVPool() + reused_req = FakeReq(req_pool_idx=3) + reused_req.kv_committed_len = 1 + new_req = FakeReq() + + req_pool_indices = alloc_req_slots( + req_to_token_pool, [reused_req, new_req], FakeTreeCache(kv_pool) + ) + + self.assertEqual(req_pool_indices, [3, 1]) + self.assertEqual(kv_pool.cleared, [[1]]) + + +if __name__ == "__main__": + unittest.main() From 45972aedf167cadb79d13c6ca42f5412e356fb5b Mon Sep 17 00:00:00 2001 From: zhangxiaolei Date: Fri, 26 Jun 2026 16:00:44 +0800 Subject: [PATCH 21/24] fix review comments --- python/sglang/srt/mem_cache/swa_radix_cache.py | 1 + 1 file changed, 1 insertion(+) diff --git a/python/sglang/srt/mem_cache/swa_radix_cache.py b/python/sglang/srt/mem_cache/swa_radix_cache.py index 1846be14bf18..b1f5de30749f 100644 --- a/python/sglang/srt/mem_cache/swa_radix_cache.py +++ b/python/sglang/srt/mem_cache/swa_radix_cache.py @@ -80,6 +80,7 @@ def __init__(self, id: Optional[int] = None): self.host_value = None # store hash values of each page self.hash_value: Optional[List[str]] = None + # for lru list, invariant: # 1. prev has greater last_access_time # 2. next has smaller last_access_time From a78423f08f7805a9e260da569a71dd785cce3f6e Mon Sep 17 00:00:00 2001 From: zhangxiaolei Date: Fri, 26 Jun 2026 16:01:40 +0800 Subject: [PATCH 22/24] Update swa_radix_cache.py --- python/sglang/srt/mem_cache/swa_radix_cache.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/sglang/srt/mem_cache/swa_radix_cache.py b/python/sglang/srt/mem_cache/swa_radix_cache.py index b1f5de30749f..c690e9636c7e 100644 --- a/python/sglang/srt/mem_cache/swa_radix_cache.py +++ b/python/sglang/srt/mem_cache/swa_radix_cache.py @@ -80,7 +80,7 @@ def __init__(self, id: Optional[int] = None): self.host_value = None # store hash values of each page self.hash_value: Optional[List[str]] = None - + # for lru list, invariant: # 1. prev has greater last_access_time # 2. next has smaller last_access_time From f5c00737949617a933789ee711d473fecf5a1f94 Mon Sep 17 00:00:00 2001 From: zhangxiaolei Date: Sun, 28 Jun 2026 22:21:00 +0800 Subject: [PATCH 23/24] Remove C128 req slot allocation reset (#620) --- python/sglang/srt/mem_cache/common.py | 43 ----------- .../srt/mem_cache/deepseek_v4_memory_pool.py | 32 +++----- .../mem_cache/test_dsv4_c128_req_state.py | 76 ------------------- 3 files changed, 11 insertions(+), 140 deletions(-) delete mode 100644 test/registered/unit/mem_cache/test_dsv4_c128_req_state.py diff --git a/python/sglang/srt/mem_cache/common.py b/python/sglang/srt/mem_cache/common.py index 58a092acdf00..0a8c83aa6c1f 100644 --- a/python/sglang/srt/mem_cache/common.py +++ b/python/sglang/srt/mem_cache/common.py @@ -49,47 +49,6 @@ logger = logging.getLogger(__name__) -def _clear_c128_req_state_for_new_slots( - tree_cache: BasePrefixCache | None, - reqs: list[Req], - already_allocated: list[bool], -) -> None: - # C128_STATE is request-scoped runtime state indexed by req_pool_idx. When a - # req slot is reused for a new request, stale partial C128 state from the - # previous request must be cleared before the new request starts using it. - # This is intentionally tied to req slot allocation instead of radix-cache - # hits: radix/HiCache hits are page-aligned and reuse complete C128 KV, while - # chunked continuation reuses an existing req slot and must keep its live - # partial state. - if tree_cache is None: - return - - allocator = getattr(tree_cache, "token_to_kv_pool_allocator", None) - if allocator is None: - return - - get_kvcache = getattr(allocator, "get_kvcache", None) - if get_kvcache is None: - return - - kv_pool = get_kvcache() - clear_c128_req_states = getattr(kv_pool, "clear_c128_req_states", None) - clear_c128_req_state = getattr(kv_pool, "clear_c128_req_state", None) - if clear_c128_req_states is None and clear_c128_req_state is None: - return - - new_req_pool_indices = [ - int(req.req_pool_idx) - for req, reused in zip(reqs, already_allocated) - if not reused and req.req_pool_idx is not None - ] - if clear_c128_req_states is not None: - clear_c128_req_states(new_req_pool_indices) - else: - for req_pool_idx in new_req_pool_indices: - clear_c128_req_state(req_pool_idx) - - def kv_to_page_indices(kv_indices: np.ndarray, page_size: int): # The page is guaranteed to be full except the last page. if page_size == 1: @@ -455,7 +414,6 @@ def alloc_req_slots( ) -> list[int]: """Allocate request slots from the pool.""" num_reqs = len(reqs) - already_allocated = [req.req_pool_idx is not None for req in reqs] if isinstance(req_to_token_pool, HybridReqToTokenPool): mamba_available_size = req_to_token_pool.mamba_allocator.available_size() if tree_cache.supports_mamba(): @@ -480,7 +438,6 @@ def alloc_req_slots( f"{req_to_token_pool.available_size()=}, " f"{num_reqs=}, " ) - _clear_c128_req_state_for_new_slots(tree_cache, reqs, already_allocated) return req_pool_indices diff --git a/python/sglang/srt/mem_cache/deepseek_v4_memory_pool.py b/python/sglang/srt/mem_cache/deepseek_v4_memory_pool.py index 9840f2be74ec..f691f564419c 100644 --- a/python/sglang/srt/mem_cache/deepseek_v4_memory_pool.py +++ b/python/sglang/srt/mem_cache/deepseek_v4_memory_pool.py @@ -946,34 +946,24 @@ def get_online_c128_mtp_pending_seq_lens(self) -> torch.Tensor: assert self.online_c128_mtp_pending_seq_lens is not None return self.online_c128_mtp_pending_seq_lens - def clear_c128_req_states(self, req_pool_indices: List[int]) -> None: - """Reset request-scoped C128 states for newly allocated req slots.""" - if not req_pool_indices: - return - + def clear_c128_req_state(self, req_pool_idx: int) -> None: + """Reset request-scoped C128 state for one req slot.""" for pool in self.compress_state_pools: if pool is None or pool.ratio != 128: continue state = pool.kv_score_buffer.kv_score - req_indices = torch.tensor( - req_pool_indices, dtype=torch.long, device=state.device - ) if ONLINE_C128: - head_dim = state.shape[-1] // 3 - state[req_indices, :head_dim] = float("-inf") - state[req_indices, head_dim:] = 0 + row = state[req_pool_idx] + head_dim = row.shape[-1] // 3 + row[:head_dim].fill_(float("-inf")) + row[head_dim:].zero_() else: - offsets = torch.arange(pool.ring_size, device=state.device) - row_indices = ( - req_indices[:, None] * pool.ring_size + offsets[None, :] - ).reshape(-1) - half = state.shape[-1] // 2 - state[row_indices, :half] = 0 - state[row_indices, half:] = float("-inf") - - def clear_c128_req_state(self, req_pool_idx: int) -> None: - self.clear_c128_req_states([req_pool_idx]) + start = req_pool_idx * pool.ring_size + rows = state[start : start + pool.ring_size] + half = rows.shape[-1] // 2 + rows[:, :half].zero_() + rows[:, half:].fill_(float("-inf")) def clear_unaccepted_c128_draft_states( self, diff --git a/test/registered/unit/mem_cache/test_dsv4_c128_req_state.py b/test/registered/unit/mem_cache/test_dsv4_c128_req_state.py deleted file mode 100644 index 322b16eb6064..000000000000 --- a/test/registered/unit/mem_cache/test_dsv4_c128_req_state.py +++ /dev/null @@ -1,76 +0,0 @@ -import unittest - -from sglang.srt.mem_cache.common import alloc_req_slots -from sglang.srt.mem_cache.memory_pool import ReqToTokenPool -from sglang.test.ci.ci_register import register_cpu_ci - -register_cpu_ci(est_time=5, suite="base-b-test-cpu") - - -class FakeKVPool: - def __init__(self): - self.cleared = [] - - def clear_c128_req_states(self, req_pool_indices): - self.cleared.append(list(req_pool_indices)) - - -class FakeAllocator: - def __init__(self, kv_pool): - self.kv_pool = kv_pool - - def get_kvcache(self): - return self.kv_pool - - -class FakeTreeCache: - def __init__(self, kv_pool): - self.token_to_kv_pool_allocator = FakeAllocator(kv_pool) - - -class FakeReq: - def __init__(self, req_pool_idx=None): - self.req_pool_idx = req_pool_idx - self.inflight_middle_chunks = 0 - self.kv_committed_len = 0 - - -class TestDSV4C128ReqState(unittest.TestCase): - def _make_pool(self): - return ReqToTokenPool( - size=4, - max_context_len=16, - device="cpu", - enable_memory_saver=False, - ) - - def test_new_req_slots_clear_c128_state(self): - req_to_token_pool = self._make_pool() - kv_pool = FakeKVPool() - reqs = [FakeReq(), FakeReq()] - - req_pool_indices = alloc_req_slots( - req_to_token_pool, reqs, FakeTreeCache(kv_pool) - ) - - self.assertEqual(req_pool_indices, [1, 2]) - self.assertEqual(kv_pool.cleared, [[1, 2]]) - - def test_reused_req_slot_skips_c128_state_clear(self): - req_to_token_pool = self._make_pool() - req_to_token_pool.free_slots.remove(3) - kv_pool = FakeKVPool() - reused_req = FakeReq(req_pool_idx=3) - reused_req.kv_committed_len = 1 - new_req = FakeReq() - - req_pool_indices = alloc_req_slots( - req_to_token_pool, [reused_req, new_req], FakeTreeCache(kv_pool) - ) - - self.assertEqual(req_pool_indices, [3, 1]) - self.assertEqual(kv_pool.cleared, [[1]]) - - -if __name__ == "__main__": - unittest.main() From bbe276483717a4da1a608e944735e3adf442f82d Mon Sep 17 00:00:00 2001 From: zhangxiaolei Date: Tue, 30 Jun 2026 10:27:15 +0800 Subject: [PATCH 24/24] Fix online c128 mtp benchmark sync pr 601 (#624) --- .../jit/benchmark/bench_online_c128_mtp.py | 12 +++--------- .../test_unified_radix_cache_kl_dsv4.py | 3 +++ 2 files changed, 6 insertions(+), 9 deletions(-) diff --git a/test/registered/jit/benchmark/bench_online_c128_mtp.py b/test/registered/jit/benchmark/bench_online_c128_mtp.py index 9f7540125ad7..f72eaf84a935 100644 --- a/test/registered/jit/benchmark/bench_online_c128_mtp.py +++ b/test/registered/jit/benchmark/bench_online_c128_mtp.py @@ -41,7 +41,6 @@ class BenchmarkCase: seq_lens: torch.Tensor req_pool_indices: torch.Tensor req_to_token: torch.Tensor - full_to_swa: torch.Tensor ape: torch.Tensor state: torch.Tensor layer_bs: int @@ -84,10 +83,6 @@ def make_case(batch_size: int, num_verify_tokens: int) -> BenchmarkCase: req_to_token = make_req_to_token(batch_size, max_seq_len, num_chunks) num_full_locs = batch_size * num_chunks - full_to_swa = ( - torch.arange(num_full_locs, dtype=torch.int64, device=DEFAULT_DEVICE) - * SWA_PAGE_SIZE - ) state_slot_stride = num_full_locs state = torch.empty( @@ -110,7 +105,6 @@ def make_case(batch_size: int, num_verify_tokens: int) -> BenchmarkCase: seq_lens=seq_lens, req_pool_indices=req_pool_indices, req_to_token=req_to_token, - full_to_swa=full_to_swa, ape=ape, state=state, layer_bs=batch_size, @@ -125,11 +119,9 @@ def call_write_prefix(module, case: BenchmarkCase) -> None: case.seq_lens, case.req_pool_indices, case.req_to_token, - case.full_to_swa, case.ape, case.state, case.layer_bs, - SWA_PAGE_SIZE, case.num_verify_tokens, case.state_slot_stride, ) @@ -151,8 +143,10 @@ def call_write_prefix(module, case: BenchmarkCase) -> None: def benchmark( batch_size: int, num_verify_tokens: int, launch_mode: str ) -> tuple[float, float, float]: - module = _jit_online_c128_mtp_module(HEAD_DIM) case = make_case(batch_size, num_verify_tokens) + module = _jit_online_c128_mtp_module( + HEAD_DIM, case.seq_lens.dtype, case.req_pool_indices.dtype + ) fn = lambda: call_write_prefix(module, case) if launch_mode == "cuda_graph": diff --git a/test/registered/radix_cache/unified_radix_tree/test_unified_radix_cache_kl_dsv4.py b/test/registered/radix_cache/unified_radix_tree/test_unified_radix_cache_kl_dsv4.py index 6c35238d362a..3e449bf13227 100644 --- a/test/registered/radix_cache/unified_radix_tree/test_unified_radix_cache_kl_dsv4.py +++ b/test/registered/radix_cache/unified_radix_tree/test_unified_radix_cache_kl_dsv4.py @@ -126,6 +126,7 @@ class TestUnifiedDeepSeekV4FlashHiCacheL3(AccuracyTwoPassMixin, CustomTestCase): l3_prefetch_page_size = 256 l3_prefetch_prompt_pages = 4 + max_running_requests = 4 @classmethod def setUpClass(cls): @@ -164,6 +165,8 @@ def setUpClass(cls): "file", "--swa-full-tokens-ratio", "0.25", + "--max-running-requests", + str(cls.max_running_requests), ], env={ "SGLANG_DSV4_FP4_EXPERTS": "0",