Skip to content
Merged
Show file tree
Hide file tree
Changes from 10 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
27 changes: 25 additions & 2 deletions python/sglang/kernels/ops/layernorm/mhc.py
Original file line number Diff line number Diff line change
Expand Up @@ -543,7 +543,15 @@ def _compute_num_split_for_mhc_pre(num_tokens: int, hc_hidden_size: int) -> int:
block_m, block_k = 64, 64
grid_size = (num_tokens + block_m - 1) // block_m
num_block_k = (hc_hidden_size + block_k - 1) // block_k
n_sms = torch.cuda.get_device_properties(0).multi_processor_count
if torch.xpu.is_available():
Comment thread
cyxlily marked this conversation as resolved.
Resolved
props = torch.xpu.get_device_properties(0)
n_sms = getattr(props, "multi_processor_count", None)
if n_sms is None:
n_sms = getattr(
props, "max_compute_units", getattr(props, "gpu_subslice_count", 1)
)
else:
n_sms = torch.cuda.get_device_properties(0).multi_processor_count
return max(1, min(n_sms // max(grid_size, 1), num_block_k // 4))


Expand Down Expand Up @@ -588,9 +596,24 @@ def prewarm_mhc_pre(
)

logger.info("DeepSeek V4 MHC prenorm prewarm: %d n_splits buckets", len(buckets))

mhc_pre_impl = mhc_pre
if residual.device.type == "xpu":
try:
from sgl_kernel import mhc_pre as xpu_mhc_pre

mhc_pre_impl = xpu_mhc_pre
logger.info("DeepSeek V4 MHC prenorm prewarm: using sgl-kernel XPU mhc_pre")
except Exception as exc:
logger.warning(
"DeepSeek V4 MHC prenorm prewarm: failed to import sgl-kernel XPU mhc_pre (%s); "
"falling back to local mhc_pre",
exc,
)
Comment thread
cyxlily marked this conversation as resolved.
Resolved

with torch.inference_mode():
for num_tokens in buckets:
mhc_pre(
mhc_pre_impl(
residual.new_zeros(num_tokens, hc_mult, hidden_size),
fn,
hc_scale,
Expand Down
66 changes: 59 additions & 7 deletions python/sglang/srt/models/deepseek_v4.py
Original file line number Diff line number Diff line change
Expand Up @@ -190,9 +190,9 @@ def _get_mhc_ops() -> MhcOps:
their communication workspaces. DeepSeek-V4 is the sole consumer here.
"""
if _is_xpu:
from sgl_kernel import hc_split_sinkhorn
from sgl_kernel import hc_split_sinkhorn, mhc_fused_post_pre

return MhcOps(hc_split_sinkhorn, None, None)
return MhcOps(hc_split_sinkhorn, mhc_fused_post_pre, None)

from sglang.kernels.ops.layernorm.mhc import (
hc_split_sinkhorn,
Expand All @@ -205,7 +205,7 @@ def _get_mhc_ops() -> MhcOps:

logger = logging.getLogger(__name__)

_FP8_WO_A_GEMM = envs.SGLANG_OPT_FP8_WO_A_GEMM.get()
_FP8_WO_A_GEMM = envs.SGLANG_OPT_FP8_WO_A_GEMM.get() and not _is_xpu
Comment thread
cyxlily marked this conversation as resolved.
Resolved
_MHC_POST_MULT_VALUE = 2.0

DEEPSEEK_V4_STACKED_PARAMS_MAPPING: List[Tuple[str, str, int]] = [
Expand All @@ -215,6 +215,11 @@ def _get_mhc_ops() -> MhcOps:


def _is_fused_mhc_post_pre_enabled() -> bool:
# CUDA/HIP fused path reuses TileLang mhc_post/mhc_pre kernels and their
# tensor layout assumptions. XPU uses the sgl-kernel SYCL fused interface.
if _is_xpu:
return envs.SGLANG_OPT_FUSE_MHC_POST_PRE.get()

# SM120 disables the standalone TileLang pre path. mhc_fused_post_pre does
# not read that flag and dispatches independently for both small and large
# token batches, so the standalone pre flag must not veto the fused opt-in.
Expand Down Expand Up @@ -1584,6 +1589,28 @@ def hc_pre_torch_impl(x, hc_fn):
)
return y, post, comb, False

if _is_xpu or x.device.type == "xpu":
from sgl_kernel import mhc_pre

norm_kwargs = {}
if norm is not None:
norm_kwargs["norm_weight"] = norm.weight.data
norm_kwargs["norm_eps"] = norm.variance_epsilon

post, comb, y = mhc_pre(
residual=x,
fn=hc_fn,
hc_scale=hc_scale,
hc_base=hc_base,
rms_eps=self.rms_norm_eps,
hc_pre_eps=self.hc_eps,
hc_sinkhorn_eps=self.hc_eps,
hc_post_mult_value=_MHC_POST_MULT_VALUE,
sinkhorn_repeat=self.hc_sinkhorn_iters,
**norm_kwargs,
)
return y, post, comb, norm is not None
Comment thread
cyxlily marked this conversation as resolved.
Resolved

if envs.SGLANG_OPT_USE_FLASHINFER_MHC.get():
y, post, comb = _flashinfer_hc_pre(
x,
Expand Down Expand Up @@ -1689,6 +1716,16 @@ def hc_post(
if _is_npu:
return torch.ops.custom.npu_hc_post(x, residual, post, comb)

if _is_xpu or x.device.type == "xpu":
from sgl_kernel import hc_post as mhc_post

return mhc_post(
x=x,
residual=residual,
post_layer_mix=post,
comb_res_mix=comb,
)

if envs.SGLANG_OPT_USE_FLASHINFER_MHC.get():
from flashinfer.mhc import mhc_post

Expand Down Expand Up @@ -2301,6 +2338,17 @@ def hc_head(
hc_base: torch.Tensor,
):
if x.numel() > 0:
if _is_xpu or x.device.type == "xpu":
from sgl_kernel import fused_hc_head

return fused_hc_head(
x.contiguous(),
hc_fn,
hc_scale,
hc_base,
norm_eps=self.norm_eps,
hc_eps=self.hc_eps,
)
from sglang.kernels.ops.layernorm.mhc_head import fused_hc_head

return fused_hc_head(
Expand Down Expand Up @@ -2850,9 +2898,13 @@ def _prewarm_mhc_kernels(self) -> None:
if self._mhc_prewarmed_at_load:
return
self._mhc_prewarmed_at_load = True
if _is_npu or not (
envs.SGLANG_DSV4_MHC_PREWARM.get()
and envs.SGLANG_OPT_USE_TILELANG_MHC_PRE.get()
if (
_is_npu
or _is_xpu
or not (
envs.SGLANG_DSV4_MHC_PREWARM.get()
and envs.SGLANG_OPT_USE_TILELANG_MHC_PRE.get()
)
):
return
layer = next(
Expand Down Expand Up @@ -2927,7 +2979,7 @@ def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]], is_nextn=Fal
else:
raise ValueError("num_nextn_predict_layers is not in the config")

if not envs.SGLANG_OPT_FP8_WO_A_GEMM.get():
if not _FP8_WO_A_GEMM:
weights = _dequant_fp8_wo_a_streaming(weights)

stacked_params_mapping = DEEPSEEK_V4_STACKED_PARAMS_MAPPING
Expand Down
149 changes: 149 additions & 0 deletions test/registered/xpu/test_hc_head.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,149 @@
import statistics
import time
import types
import unittest

import pytest
import torch

from sglang.kernels.ops.layernorm.mhc_head import fused_hc_head as triton_fused_hc_head
from sglang.srt.models.deepseek_v4 import DeepseekV4Model
from sglang.test.ci.ci_register import register_xpu_ci
from sglang.test.test_utils import CustomTestCase

register_xpu_ci(est_time=30, suite="stage-b-test-1-gpu-xpu")


@pytest.fixture(autouse=True)
def skip_if_no_xpu():
if not hasattr(torch, "xpu") or not torch.xpu.is_available():
pytest.skip("XPU not available")
Comment thread
cyxlily marked this conversation as resolved.
Resolved


def _sync_xpu() -> None:
if hasattr(torch, "xpu") and torch.xpu.is_available():
torch.xpu.synchronize()
Comment thread
cyxlily marked this conversation as resolved.
Resolved


def _bench_ms(fn, warmup: int = 10, iters: int = 50):
for _ in range(warmup):
fn()
_sync_xpu()

samples = []
for _ in range(iters):
start = time.perf_counter()
fn()
_sync_xpu()
samples.append((time.perf_counter() - start) * 1000.0)

return {
"mean_ms": statistics.mean(samples),
"median_ms": statistics.median(samples),
"min_ms": min(samples),
"max_ms": max(samples),
}
Comment thread
cyxlily marked this conversation as resolved.
Resolved


class TestHCHead(CustomTestCase):
HC_MULT = 4
HIDDEN_SIZE = 7168
NUM_TOKENS = [1, 16, 128, 1024]
DTYPE = torch.bfloat16
NORM_EPS = 1e-6
HC_EPS = 1e-6
SEED = 2026

@classmethod
def setUpClass(cls):
if not hasattr(torch, "xpu") or not torch.xpu.is_available():
raise unittest.SkipTest("XPU is not available")
try:
from sgl_kernel.mhc import fused_hc_head as _ # noqa: F401
except ImportError:
raise unittest.SkipTest("sgl-kernel fused_hc_head is not available")
torch.set_default_device("xpu")

def _make_inputs(self, num_tokens: int):
torch.manual_seed(self.SEED + num_tokens)
x = torch.randn(
num_tokens,
self.HC_MULT,
self.HIDDEN_SIZE,
dtype=self.DTYPE,
device="xpu",
).contiguous()
hc_fn = torch.randn(
self.HC_MULT,
self.HC_MULT * self.HIDDEN_SIZE,
dtype=torch.float32,
device="xpu",
).contiguous()
hc_base = torch.randn(
self.HC_MULT,
dtype=torch.float32,
device="xpu",
).contiguous()
hc_scale = torch.randn(1, dtype=torch.float32, device="xpu").contiguous()
return x, hc_fn, hc_scale, hc_base

def test_hc_head(self):
for num_tokens in self.NUM_TOKENS:
with self.subTest(num_tokens=num_tokens):
x, hc_fn, hc_scale, hc_base = self._make_inputs(num_tokens)

_ctx = types.SimpleNamespace(norm_eps=self.NORM_EPS, hc_eps=self.HC_EPS)

with torch.inference_mode():
triton_out = triton_fused_hc_head(
x,
hc_fn,
hc_scale,
hc_base,
self.NORM_EPS,
self.HC_EPS,
)
# Go through the actual DeepseekV4Model.hc_head dispatch path
# (on XPU this calls sgl_kernel.mhc.fused_hc_head).
sycl_out = DeepseekV4Model.hc_head(
_ctx, x, hc_fn, hc_scale, hc_base
)

torch.testing.assert_close(
triton_out,
sycl_out,
atol=2e-2,
rtol=2e-2,
)

def test_hc_head_perf(self):
for num_tokens in self.NUM_TOKENS:
with self.subTest(num_tokens=num_tokens):
x, hc_fn, hc_scale, hc_base = self._make_inputs(num_tokens)

_ctx = types.SimpleNamespace(norm_eps=self.NORM_EPS, hc_eps=self.HC_EPS)

triton_stats = _bench_ms(
lambda: triton_fused_hc_head(
x,
hc_fn,
hc_scale,
hc_base,
self.NORM_EPS,
self.HC_EPS,
)
)
sycl_stats = _bench_ms(
lambda: DeepseekV4Model.hc_head(_ctx, x, hc_fn, hc_scale, hc_base)
)

self.assertLess(
sycl_stats["median_ms"],
triton_stats["median_ms"],
f"sycl ({sycl_stats['median_ms']:.3f} ms) not faster than triton "
f"({triton_stats['median_ms']:.3f} ms) at num_tokens={num_tokens}",
)
Comment thread
cyxlily marked this conversation as resolved.
Resolved


if __name__ == "__main__":
unittest.main()
Loading