Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions python/pyproject.toml
Original file line number Diff line number Diff line change
Expand Up @@ -76,6 +76,7 @@ dependencies = [
"setproctitle",
"sgl-deep-ep==0.1.2.post1",
"sgl-deep-gemm==0.2.3",
"sgl-flash-mla==1.0.0",
"sglang-kernel==0.4.10",
"smg-grpc-servicer>=0.9.0",
"soundfile==0.13.1",
Expand Down
4 changes: 2 additions & 2 deletions python/sglang/kernels/jit/csrc/deepseek_v4/c1.cuh
Original file line number Diff line number Diff line change
Expand Up @@ -162,7 +162,7 @@ __global__ __launch_bounds__(kHeadDim / kC1VecSize) void flash_c1_decode_kernel(
}
}

if constexpr (kLayout == KVLayout::V41_FP4) {
if constexpr (deepseek_v4::kv_encoding(kLayout) == KVLayout::V41_FP4) {
// The fp4 cache takes the rotated bf16 value as is: its row quantizer is the fake quant, minus the dequant.
if (out_loc <= 0) return;
const auto kv_row = Paged::row(params.kvcache, out_loc);
Expand Down Expand Up @@ -191,7 +191,7 @@ __global__ __launch_bounds__(kHeadDim / kC1VecSize) void flash_c1_decode_kernel(
if (out_loc <= 0) return;
const auto kv_row = Paged::row(params.kvcache, out_loc);

if constexpr (kLayout == KVLayout::V41 || kLayout == KVLayout::UNIFORM_FP8) {
if constexpr (deepseek_v4::kv_encoding(kLayout) == KVLayout::V41 || kLayout == KVLayout::UNIFORM_FP8) {
// FP8 over the whole row, RoPE included; the layout selects the scaling.
return deepseek_v4::v41::store_row<kLayout>(kv_row.data, kv_row.scale, tx, data);
}
Expand Down
4 changes: 2 additions & 2 deletions python/sglang/kernels/jit/csrc/deepseek_v4/c2.cuh
Original file line number Diff line number Diff line change
Expand Up @@ -232,7 +232,7 @@ __global__ __launch_bounds__(kHeadDim / kC2VecSize) void flash_c2_decode_kernel(
}

const int32_t out_loc = static_cast<uint32_t>(raw_out_loc) >> 1;
if constexpr (kLayout == KVLayout::V41_FP4) {
if constexpr (deepseek_v4::kv_encoding(kLayout) == KVLayout::V41_FP4) {
// The fp4 cache takes the rotated bf16 value as is: its row quantizer is the fake quant, minus the dequant.
const auto kv_row = Paged::row(params.kvcache, out_loc);
return deepseek_v4::v41::store_row<kLayout>(kv_row.data, kv_row.scale, tx, staged);
Expand All @@ -258,7 +258,7 @@ __global__ __launch_bounds__(kHeadDim / kC2VecSize) void flash_c2_decode_kernel(
// `raw_out_loc / ratio`; ratio 2 makes it a shift.
const auto kv_row = Paged::row(params.kvcache, out_loc);

if constexpr (kLayout == KVLayout::V41 || kLayout == KVLayout::UNIFORM_FP8) {
if constexpr (deepseek_v4::kv_encoding(kLayout) == KVLayout::V41 || kLayout == KVLayout::UNIFORM_FP8) {
// FP8 over the whole row, RoPE included; the layout selects the scaling.
return deepseek_v4::v41::store_row<kLayout>(kv_row.data, kv_row.scale, tx, staged);
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -16,9 +16,8 @@
// critical path, and it cannot be hoisted out because the schedule depends on
// the per-request `topk_length` of the step being replayed.
//
// This kernel produces the same output bit for bit; the shared-memory layout
// matches FlashMLA's exactly because the idle-tail fill below reads one entry
// past `first_block_idx_shared`, as FlashMLA does.
// This kernel produces the same output bit for bit as the legacy scheduler in
// sgl-flash-mla 1.0.0, idle-tail partitions included.

namespace sglang {

Expand Down Expand Up @@ -180,9 +179,8 @@ __global__ void __launch_bounds__(kBlockSize) flashmla_sched_meta_kernel(__grid_
const int lb_last = last_block_idx_shared[b - 1];
const int sk_last = seqlens_k_shared[b - 1];
const int end_block_idx = (sk_last == 0) ? 0 : lb_last + 1;
// FlashMLA reads first_block_idx_shared[batch_size] for these, one past the
// end of that array, which aliases last_block_idx_shared[0].
const int begin_block_idx = last_block_idx_shared[0];
// `now_block` is 0 once every request is consumed.
const int begin_block_idx = 0;
const int is_last_req_splitted = (end_block_idx != lb_last + 1) && (sk_last != 0);
for (int i = first_idle_part_shared + threadIdx.x; i < p.num_sm_parts; i += kBlockSize) {
int* meta = out_shared + i * kMetaInts;
Expand Down
2 changes: 1 addition & 1 deletion python/sglang/kernels/jit/csrc/deepseek_v4/store.cuh
Original file line number Diff line number Diff line change
Expand Up @@ -84,7 +84,7 @@ __global__ void fused_store_flashmla_cache(const __grid_constant__ FusedStoreCac
/// Elements per thread of the V4.1 store (`512 / vec` threads per token): 4 for fp8 rows, 2 for
/// fp4, whose per-element IEEE divisions spread better over more threads (measured on B200, bs 1..512).
constexpr uint32_t v41_store_vec_size(deepseek_v4::KVLayout layout) {
return layout == deepseek_v4::KVLayout::V41 ? 4 : 2;
return deepseek_v4::kv_encoding(layout) == deepseek_v4::KVLayout::V41 ? 4 : 2;
}

template <
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -16,8 +16,8 @@
#endif

// Paged fp8 / fp4 KV cache layouts read by the d_qk = 512 sparse MLA decode
// kernels. A page block is `page_size` data rows followed by `page_size` scale
// rows, so the scale region starts at byte `page_size * kDataBytes`:
// kernels. A block-split page is `page_size` data rows followed by `page_size`
// scale rows, so the scale region starts at byte `page_size * kDataBytes`:
//
// V4 584 B/token: 448 e4m3 + 64 bf16 (RoPE) data, 7 ue8m0 scales + 1 pad,
// one scale per 64 e4m3 values.
Expand All @@ -26,16 +26,27 @@
// V41_FP4 288 B/token: 512 e2m1 data packed two per byte (even index in the
// low nibble), 32 e4m3 scales, one per 16 values.
//
// The reader requires the rows of a page to be contiguous and the page stride
// to be a multiple of kPageAlign (its TMA row stride), which is what
// kv_page_bytes pads to. The pure-torch reference of the V41 (fp8) quantizer is
// `sglang.test.kernels.deepseek_v4.k_cache_reference`.
// V41_NEW / V41_FP4_NEW (FlashMLA's 2026.09.30 kernels) store the V41 / V41_FP4
// bytes interleaved: each token is its data immediately followed by its scales,
// 528 / 288 B apart, with no page padding.
//
// The block-split reader requires the rows of a page to be contiguous and the
// page stride to be a multiple of kPageAlign (its TMA row stride), which is what
// kv_page_bytes pads to. The pure-torch reference of the V41 quantizers is
// `sglang.test.kernels.deepseek_v4.dsv41_kv_quant_reference`.

namespace sglang {

namespace deepseek_v4 {

enum class KVLayout : int32_t { V4 = 0, V41 = 1, V41_FP4 = 2, UNIFORM_FP8 = 3 };
enum class KVLayout : int32_t { V4 = 0, V41 = 1, V41_FP4 = 2, UNIFORM_FP8 = 3, V41_NEW = 4, V41_FP4_NEW = 5 };

/// The token encoding of a layout: V41_NEW / V41_FP4_NEW store the V41 / V41_FP4 bytes.
constexpr KVLayout kv_encoding(KVLayout layout) {
if (layout == KVLayout::V41_NEW) return KVLayout::V41;
if (layout == KVLayout::V41_FP4_NEW) return KVLayout::V41_FP4;
return layout;
}

template <KVLayout kLayout>
struct KVLayoutTraits;
Expand All @@ -46,6 +57,7 @@ struct KVLayoutTraits<KVLayout::V4> {
static constexpr int64_t kScaleBytes = 8;
static constexpr int64_t kTileSize = 64;
static constexpr int64_t kPageAlign = 576;
static constexpr bool kInterleaved = false;
static constexpr int64_t kBytesPerToken = kDataBytes + kScaleBytes;
};

Expand All @@ -55,6 +67,7 @@ struct KVLayoutTraits<KVLayout::V41> {
static constexpr int64_t kScaleBytes = 16;
static constexpr int64_t kTileSize = 32;
static constexpr int64_t kPageAlign = 512;
static constexpr bool kInterleaved = false;
static constexpr int64_t kBytesPerToken = kDataBytes + kScaleBytes;
};

Expand All @@ -64,6 +77,7 @@ struct KVLayoutTraits<KVLayout::V41_FP4> {
static constexpr int64_t kScaleBytes = 32;
static constexpr int64_t kTileSize = 16;
static constexpr int64_t kPageAlign = 256;
static constexpr bool kInterleaved = false;
static constexpr int64_t kBytesPerToken = kDataBytes + kScaleBytes;
};

Expand All @@ -73,24 +87,39 @@ struct KVLayoutTraits<KVLayout::UNIFORM_FP8> {
static constexpr int64_t kScaleBytes = 0;
static constexpr int64_t kTileSize = 512;
static constexpr int64_t kPageAlign = 512;
static constexpr bool kInterleaved = false;
static constexpr int64_t kBytesPerToken = kDataBytes;
};

template <KVLayout kLayout>
struct InterleavedKVLayoutTraits : KVLayoutTraits<kv_encoding(kLayout)> {
static constexpr bool kInterleaved = true;
static constexpr int64_t kPageAlign = KVLayoutTraits<kv_encoding(kLayout)>::kBytesPerToken;
};

template <>
struct KVLayoutTraits<KVLayout::V41_NEW> : InterleavedKVLayoutTraits<KVLayout::V41_NEW> {};

template <>
struct KVLayoutTraits<KVLayout::V41_FP4_NEW> : InterleavedKVLayoutTraits<KVLayout::V41_FP4_NEW> {};

/// Bytes of one page block: `page_size` tokens, padded up to the reader's row stride.
template <KVLayout kLayout>
constexpr int64_t kv_page_bytes(int64_t page_size) {
using Traits = KVLayoutTraits<kLayout>;
return (page_size * Traits::kBytesPerToken + Traits::kPageAlign - 1) / Traits::kPageAlign * Traits::kPageAlign;
}

/// Addressing of a paged cache: `1 << kPageBits` data rows then as many scale rows, padded to kPageAlign.
/// Addressing of a paged cache of `1 << kPageBits` tokens per page, padded to kPageAlign:
/// slot `i`'s data at `i * kDataStride`, its scales at `kScaleBase + i * kScaleStride`.
template <KVLayout kLayout, uint32_t kPageBits>
struct PagedKV {
using Traits = KVLayoutTraits<kLayout>;
static constexpr int64_t kPageSize = int64_t{1} << kPageBits;
static constexpr int64_t kPageBytes = kv_page_bytes<kLayout>(kPageSize);
/// Byte offset of the scale rows inside a page.
static constexpr int64_t kScaleBase = Traits::kDataBytes << kPageBits;
static constexpr int64_t kDataStride = Traits::kInterleaved ? Traits::kBytesPerToken : Traits::kDataBytes;
static constexpr int64_t kScaleBase = Traits::kInterleaved ? Traits::kDataBytes : Traits::kDataBytes << kPageBits;
static constexpr int64_t kScaleStride = Traits::kInterleaved ? Traits::kBytesPerToken : Traits::kScaleBytes;

template <typename LocT>
static constexpr LocT page_of(LocT loc) {
Expand All @@ -103,11 +132,11 @@ struct PagedKV {
/// Byte offsets of token `loc`'s data row and scale row from the cache base.
template <typename LocT>
static constexpr int64_t data_offset(LocT loc) {
return page_of(loc) * kPageBytes + slot_of(loc) * Traits::kDataBytes;
return page_of(loc) * kPageBytes + slot_of(loc) * kDataStride;
}
template <typename LocT>
static constexpr int64_t scale_offset(LocT loc) {
return page_of(loc) * kPageBytes + kScaleBase + slot_of(loc) * Traits::kScaleBytes;
return page_of(loc) * kPageBytes + kScaleBase + slot_of(loc) * kScaleStride;
}

struct Row {
Expand All @@ -118,7 +147,7 @@ struct PagedKV {
template <typename LocT>
SGL_DEVICE static Row row(uint8_t* cache, LocT loc) {
uint8_t* page = cache + page_of(loc) * kPageBytes;
return {page + slot_of(loc) * Traits::kDataBytes, page + kScaleBase + slot_of(loc) * Traits::kScaleBytes};
return {page + slot_of(loc) * kDataStride, page + kScaleBase + slot_of(loc) * kScaleStride};
}
};

Expand Down Expand Up @@ -213,7 +242,7 @@ SGL_DEVICE void store_row_fp4(uint8_t* data_row, uint8_t* scale_row, uint32_t tx
template <KVLayout kLayout, uint32_t kVecSize>
SGL_DEVICE void store_row(uint8_t* data_row, uint8_t* scale_row, uint32_t tx, const float (&v)[kVecSize]) {
static_assert(kLayout != KVLayout::V4, "V4 rows are written by the caller");
if constexpr (kLayout == KVLayout::V41) {
if constexpr (kv_encoding(kLayout) == KVLayout::V41) {
store_row_fp8(data_row, scale_row, tx, v);
} else if constexpr (kLayout == KVLayout::UNIFORM_FP8) {
device::AlignedVector<fp8x2_e4m3_t, kVecSize / 2> out;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@
read as ``kv[:, :d_v]``, so there are no partials and no merge pass.
``dsa/triton_sparse_mla.py`` is the gfx950-only FP8 sibling of this bf16 module.
Interface matches ``sgl_kernel.flash_mla.flash_mla_sparse_fwd``::
Interface matches ``sglang.kernels.ops.attention.flash_mla_interface.flash_mla_sparse_fwd``::
q [T, H, 576] bf16 (absorbed MLA: 512 nope + 64 rope)
kv [S, 576] bf16 (V is kv[:, :512])
Expand Down
27 changes: 17 additions & 10 deletions python/sglang/kernels/ops/attention/dsv4/dequant_k_cache.py
Original file line number Diff line number Diff line change
Expand Up @@ -102,13 +102,13 @@ def dequantize_k_cache_paged_v41(
out: Optional[torch.Tensor] = None,
layout: KVLayout = KVLayout.V41,
) -> torch.Tensor:
"""Dequantize a V4.1 paged cache (fp8 ``V41`` or fp4 ``V41_FP4``) for a list
"""Dequantize a V4.1 paged cache (fp8 ``V41`` / ``V41_NEW`` or fp4 ``V41_FP4`` / ``V41_FP4_NEW``) for a list
of token IDs into ``(num_tokens, 1, 512)`` bf16.
Bit-exact with the pure-torch dequantizer of these formats.
"""
layout = KVLayout.parse(layout)
assert layout in (KVLayout.V41, KVLayout.V41_FP4), layout
assert layout.encoding in (KVLayout.V41, KVLayout.V41_FP4), layout
if is_hip_runtime():
supported = is_gfx95_supported()
else:
Expand Down Expand Up @@ -143,9 +143,10 @@ def dequantize_k_cache_paged_v41(

kernel = (
_dequantize_k_cache_paged_v41_fp8_kernel
if layout is KVLayout.V41
if layout.encoding is KVLayout.V41
else _dequantize_k_cache_paged_v41_fp4_kernel
)
data_stride, scale_base, scale_stride = layout.row_addressing(page_size)
kernel[(num_tokens,)](
out,
buf_fp8,
Expand All @@ -157,7 +158,9 @@ def dequantize_k_cache_paged_v41(
DATA_BYTES=layout.data_bytes,
SCALE_BYTES=layout.scale_bytes,
TILE_SIZE=layout.tile_size,
S_OFFSET_BYTES=layout.scale_offset(page_size),
DATA_STRIDE=data_stride,
SCALE_BASE=scale_base,
SCALE_STRIDE=scale_stride,
IS_HIP=is_hip_runtime(),
)
return out
Expand Down Expand Up @@ -329,7 +332,9 @@ def _dequantize_k_cache_paged_v41_fp8_kernel(
DATA_BYTES: tl.constexpr,
SCALE_BYTES: tl.constexpr,
TILE_SIZE: tl.constexpr,
S_OFFSET_BYTES: tl.constexpr,
DATA_STRIDE: tl.constexpr,
SCALE_BASE: tl.constexpr,
SCALE_STRIDE: tl.constexpr,
IS_HIP: tl.constexpr,
):
# V41: 512 e4m3 values per token, then 16 ue8m0 scales (one per 32 values).
Expand All @@ -339,8 +344,8 @@ def _dequantize_k_cache_paged_v41_fp8_kernel(
page_idx = loc // PAGE_SIZE
in_page = loc % PAGE_SIZE
page_byte_base = page_idx * BYTES_PER_PAGE
token_data_base = page_byte_base + in_page * DATA_BYTES
token_scale_base = page_byte_base + S_OFFSET_BYTES + in_page * SCALE_BYTES
token_data_base = page_byte_base + in_page * DATA_STRIDE
token_scale_base = page_byte_base + SCALE_BASE + in_page * SCALE_STRIDE

offs = tl.arange(0, DATA_BYTES)
vals = tl.load(buf_fp8_ptr + token_data_base + offs).to(tl.float32)
Expand All @@ -364,7 +369,9 @@ def _dequantize_k_cache_paged_v41_fp4_kernel(
DATA_BYTES: tl.constexpr,
SCALE_BYTES: tl.constexpr,
TILE_SIZE: tl.constexpr,
S_OFFSET_BYTES: tl.constexpr,
DATA_STRIDE: tl.constexpr,
SCALE_BASE: tl.constexpr,
SCALE_STRIDE: tl.constexpr,
IS_HIP: tl.constexpr,
):
# V41_FP4: 512 e2m1 codes packed two per byte (even index in the low nibble),
Expand All @@ -375,8 +382,8 @@ def _dequantize_k_cache_paged_v41_fp4_kernel(
page_idx = loc // PAGE_SIZE
in_page = loc % PAGE_SIZE
page_byte_base = page_idx * BYTES_PER_PAGE
token_data_base = page_byte_base + in_page * DATA_BYTES
token_scale_base = page_byte_base + S_OFFSET_BYTES + in_page * SCALE_BYTES
token_data_base = page_byte_base + in_page * DATA_STRIDE
token_scale_base = page_byte_base + SCALE_BASE + in_page * SCALE_STRIDE

boffs = tl.arange(0, DATA_BYTES)
packed = tl.load(buf_uint8_ptr + token_data_base + boffs)
Expand Down
Loading
Loading