Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
16 changes: 8 additions & 8 deletions python/sglang/srt/layers/quantization/fp8_utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -1937,24 +1937,24 @@ def _inverse_transform_scale_ue8m0_impl(sf_packed):
assert len(sf_packed.shape) == 2, f"{sf_packed.shape=}"
assert sf_packed.dtype == torch.int32

mn_repeat_128, k_div_4 = sf_packed.shape
mn = mn_repeat_128 // block_size
# one row per weight row, so the last block is partial when the rows are not a multiple of 128
num_rows, k_div_4 = sf_packed.shape
mn = -(-num_rows // block_size)
k = k_div_4 * 4

# packed u8 -> fp32
sf_u8 = sf_packed.contiguous().flatten().view(torch.uint8).view(mn_repeat_128, k)
sf_u8 = sf_packed.contiguous().flatten().view(torch.uint8).view(num_rows, k)
sf_fp32 = (sf_u8.to(torch.int32) << 23).view(torch.float32)

# remove repeat
sf_reshaped = sf_fp32.view(mn, block_size, k)
sf_unrepeated = sf_reshaped[:, 0:1, :]
if not torch.all(sf_unrepeated == sf_reshaped):
sf_unrepeated = sf_fp32[::block_size].contiguous()
sf_repeated = sf_unrepeated.repeat_interleave(block_size, dim=0)[:num_rows]
if not torch.all(sf_repeated == sf_fp32):
from sglang.srt.debug_utils.dumper import get_tensor_info

raise AssertionError(
f"sf_unrepeated != sf_reshaped ({get_tensor_info(sf_unrepeated)=} {get_tensor_info(sf_reshaped)=})"
f"sf_repeated != sf_fp32 ({get_tensor_info(sf_repeated)=} {get_tensor_info(sf_fp32)=})"
)
sf_unrepeated = sf_unrepeated.squeeze(1).contiguous()

assert sf_unrepeated.shape == (mn, k)
return sf_unrepeated
Expand Down
3 changes: 3 additions & 0 deletions python/sglang/srt/layers/quantization/modelopt_quant.py
Original file line number Diff line number Diff line change
Expand Up @@ -2574,6 +2574,9 @@ def create_weights(
)
w2_input_scale._sglang_require_global_experts = True
layer.register_parameter("w2_input_scale", w2_input_scale)
# calibrated activation scales, as the KV cache scales: no weight update carries them
w13_input_scale._skip_weight_check = True
w2_input_scale._skip_weight_check = True

def _build_mega_moe_weights(self, layer: torch.nn.Module) -> None:
# Activations are quantized per token at dispatch, so w13_input_scale
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -94,6 +94,11 @@ def _resolve_eagle_aux_hidden_state(
is_draft_model=True,
)
num_nextn_predict_layers = draft_model_config.num_nextn_predict_layers
if num_nextn_predict_layers is None:
# Qwen3.5 names its MTP depth mtp_num_hidden_layers; models without MTP have neither
num_nextn_predict_layers = getattr(
draft_model_config.hf_text_config, "mtp_num_hidden_layers", None
)
if num_nextn_predict_layers is not None:
config.eagle_draft_num_layers = int(num_nextn_predict_layers)
elif get_spec().speculative_draft_model_path:
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@
DefaultModelLoader,
get_model_loader,
post_load_weights,
refresh_gemma_weights,
)
from sglang.srt.model_loader.utils import set_default_torch_dtype
from sglang.srt.model_loader.weight_utils import default_weight_loader
Expand Down Expand Up @@ -306,6 +307,8 @@ def begin_weight_update(self: WeightUpdater) -> None:
def end_weight_update(self: WeightUpdater, *, run_post_load: bool) -> None:
if run_post_load:
post_load_weights(self.get_model())
else:
refresh_gemma_weights(self.get_model())
DefaultModelLoader.postprocess_weights(
self.get_model(), torch.device(self.device)
)
Expand Down
6 changes: 5 additions & 1 deletion python/sglang/srt/model_loader/loader.py
Original file line number Diff line number Diff line change
Expand Up @@ -326,7 +326,11 @@ def post_load_weights(model: nn.Module) -> None:
# `is_nextn=True`, so the loader doesn't need to know.
if hasattr(model, "post_load_weights"):
model.post_load_weights()
# its weight loader derives the buffer, and these loaders bypass it
refresh_gemma_weights(model)


def refresh_gemma_weights(model: nn.Module) -> None:
# its weight loader derives the buffer, so writes that bypass the loader leave it stale
for module in model.modules():
if isinstance(module, GemmaRMSNorm):
module.refresh_gemma_weight()
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -39,6 +39,7 @@
from sglang.srt.layers.quantization.int8_utils import (
block_dequant as int8_block_dequant,
)
from sglang.srt.layers.quantization.mxfp8_block_convert import dequant_mxfp8_2d_to_bf16
from sglang.srt.layers.utils import get_layer_id
from sglang.srt.model_loader.utils import (
maybe_executor_submit,
Expand Down Expand Up @@ -86,6 +87,19 @@ def _get_indexer_weight_block_size(
return [128, 128]


def _dequant_fused_indexer_shard(
weight: torch.Tensor,
scale: torch.Tensor,
quant_config: Optional[QuantizationConfig],
) -> torch.Tensor:
# mxfp8 scales are UE8M0 exponents stored as uint8, not fp32 multipliers
if scale.dtype == torch.uint8:
return dequant_mxfp8_2d_to_bf16(weight, scale)
return block_quant_dequant(
weight, scale, _get_indexer_weight_block_size(quant_config), torch.bfloat16
)


def _load_fused_indexer_wk(
name: str,
loaded_weight: torch.Tensor,
Expand Down Expand Up @@ -119,9 +133,8 @@ def _load_fused_indexer_wk(
)
if "weight" in entry and "scale" in entry:
pending.pop(fused_name + ".weights_proj")
block_size = _get_indexer_weight_block_size(quant_config)
weights_bf16 = block_quant_dequant(
entry["weight"], entry["scale"], block_size, torch.bfloat16
weights_bf16 = _dequant_fused_indexer_shard(
entry["weight"], entry["scale"], quant_config
)
fused_param.data[-weights_bf16.shape[0] :].copy_(weights_bf16)
return True
Expand All @@ -139,9 +152,8 @@ def _load_fused_indexer_wk(
)
if "weight" in entry and "scale" in entry:
pending.pop(fused_name)
block_size = _get_indexer_weight_block_size(quant_config)
wk_bf16 = block_quant_dequant(
entry["weight"], entry["scale"], block_size, torch.bfloat16
wk_bf16 = _dequant_fused_indexer_shard(
entry["weight"], entry["scale"], quant_config
)
fused_param.data[: wk_bf16.shape[0]].copy_(wk_bf16)
return True
Expand Down
17 changes: 11 additions & 6 deletions test/registered/unit/layers/quantization/test_fp8_utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -82,13 +82,18 @@ def is_deep_gemm(self):

class TestInverseTransformScaleUe8m0(CustomTestCase):
def test_round_trip(self):
for shape in (
# DeepSeek V3 kv_b_proj
(32768, 512),
# GLM-5 fused q_a + kv_a: 2048 + 576 rows, the last block partial
(2624, 6144),
):
with self.subTest(shape=shape):
self._check_round_trip(shape)

def _check_round_trip(self, shape):
for _ in range(100):
weight_bf16 = torch.randn(
# DeepSeek V3 kv_b_proj
(32768, 512),
dtype=torch.bfloat16,
device="cuda",
)
weight_bf16 = torch.randn(shape, dtype=torch.bfloat16, device="cuda")

weight_block_size = [128, 128]

Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,81 @@
"""The weight checker must leave the NVFP4 MoE activation input scales alone: no weight update carries them, so
a reset before an update leaves them random and the comparison after it fails whatever the update wrote.

The platform check is stubbed so the layer is built on CPU, as in test_modelopt_nvfp4_moe_dispatch.py.
"""

from sglang.test.ci.ci_register import register_cpu_ci

register_cpu_ci(est_time=6, suite="base-a-test-cpu")

import unittest
from types import SimpleNamespace
from unittest import mock

import torch

# Import modelopt_quant first (see test_modelopt_nvfp4_moe_scales.py for the circular-import reason).
# isort: off
from sglang.srt.layers.quantization import modelopt_quant
from sglang.srt.layers.quantization.modelopt_quant import ModelOptNvFp4FusedMoEMethod

# isort: on
from sglang.srt.layers.moe.utils import MoeRunnerBackend
from sglang.srt.utils.weight_checker import _is_skip_weight_check
from sglang.test.test_utils import CustomTestCase

NUM_EXPERTS = 8
HIDDEN = 64
INTERMEDIATE = 32


def _nvfp4_moe_layer() -> torch.nn.Module:
with (
mock.patch.object(
modelopt_quant,
"get_moe_runner_backend",
return_value=MoeRunnerBackend.FLASHINFER_TRTLLM_ROUTED,
),
mock.patch.object(
modelopt_quant,
"get_platform",
return_value=SimpleNamespace(is_blackwell=True),
),
mock.patch.object(modelopt_quant, "is_cuda", return_value=False),
):
method = ModelOptNvFp4FusedMoEMethod(
SimpleNamespace(
group_size=16,
is_checkpoint_nvfp4_serialized=True,
use_per_token_activation=False,
get_name=lambda: "modelopt_fp4",
)
)
layer = torch.nn.Module()
layer.num_experts = NUM_EXPERTS
layer.num_local_experts = NUM_EXPERTS
layer.moe_ep_rank = 0
layer.moe_runner_config = SimpleNamespace(is_gated=True)
method.create_weights(
layer,
num_experts=NUM_EXPERTS,
hidden_size=HIDDEN,
intermediate_size_per_partition=INTERMEDIATE,
params_dtype=torch.bfloat16,
weight_loader=lambda *args, **kwargs: None,
)
return layer


class TestNvFp4MoeWeightCheck(CustomTestCase):
def test_the_checker_skips_the_input_scales_and_checks_the_weight_scales(self):
params = dict(_nvfp4_moe_layer().named_parameters())

for name in ("w13_input_scale", "w2_input_scale"):
self.assertTrue(_is_skip_weight_check(name, params[name]), name)
for name in ("w13_weight", "w13_weight_scale", "w13_weight_scale_2"):
self.assertFalse(_is_skip_weight_check(name, params[name]), name)


if __name__ == "__main__":
unittest.main()
Original file line number Diff line number Diff line change
Expand Up @@ -3,9 +3,13 @@

import pytest

from sglang.srt.model_executor.model_runner_components import spec_aux_hidden_state
from sglang.srt.model_executor.model_runner_components.spec_aux_hidden_state import (
SpecAuxHiddenStateConfig,
_map_muse_target_layer_ids,
_resolve_eagle_aux_hidden_state,
)
from sglang.srt.speculative.spec_info import SpeculativeAlgorithm
from sglang.test.ci.ci_register import register_cpu_ci

register_cpu_ci(est_time=11, suite="base-a-test-cpu")
Expand Down Expand Up @@ -34,5 +38,34 @@ def test_muse_target_layer_id_mapping(target_model_type, draft_architecture, exp
)


def test_an_mtp_depth_named_mtp_num_hidden_layers_counts_as_the_eagle_draft(
monkeypatch,
):
"""Qwen3.5 names its MTP depth mtp_num_hidden_layers. Read as no draft, the target's
KV budget leaves the draft's pool, of the target's token count, outside
mem_fraction_static."""
monkeypatch.setattr(
spec_aux_hidden_state,
"get_spec",
lambda: SimpleNamespace(speculative_draft_model_path=None),
)
config = SpecAuxHiddenStateConfig()

_resolve_eagle_aux_hidden_state(
config=config,
server_args=None,
model_config=SimpleNamespace(
num_nextn_predict_layers=None,
hf_text_config=SimpleNamespace(mtp_num_hidden_layers=1),
is_hybrid_swa=False,
is_deepseek_v4_arch=False,
),
spec_algorithm=SpeculativeAlgorithm.EAGLE,
is_draft_worker=False,
)

assert config.eagle_draft_num_layers == 1


if __name__ == "__main__":
sys.exit(pytest.main([__file__, "-v"]))
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
"""A weight-update session must leave every parameter that peers write by address where it was published."""
"""A weight-update session must leave every parameter that peers write by address where it was published, and
rebuild what the loaders derive from the weights it wrote."""

import unittest
from contextlib import contextmanager
Expand All @@ -8,6 +9,7 @@
import torch

import sglang.srt.model_executor.model_runner_components.weight_updater as weight_updater_mod
from sglang.srt.layers.layernorm import GemmaRMSNorm
from sglang.srt.model_executor.model_runner_components.weight_updater import (
WeightUpdater,
)
Expand Down Expand Up @@ -96,5 +98,22 @@ def test_a_rank_that_published_nothing_is_not_checked(self):
weight_updater.end_weight_update(run_post_load=False)


class TestDerivedState(CustomTestCase):
def test_gemma_weight_follows_a_norm_weight_loaded_without_its_loader(self):
"""`direct` and custom loads write params without GemmaRMSNorm's loader yet mark the session loaded, so
fused allreduce kernels would read the old `gemma_weight`."""
model = torch.nn.Module()
model.norm = GemmaRMSNorm(4)
weight_updater = _weight_updater(model, published_weight_info=None)
new_weight = torch.tensor([0.5, -0.25, 0.0, 2.0])

with _postprocess(lambda model, device: None):
weight_updater.begin_weight_update()
model.norm.weight.data.copy_(new_weight)
weight_updater.end_weight_update(run_post_load=False)

torch.testing.assert_close(model.norm.gemma_weight, new_weight + 1)


if __name__ == "__main__":
unittest.main()
53 changes: 53 additions & 0 deletions test/registered/unit/models/test_deepseek_fused_indexer_mxfp8.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,53 @@
"""An mxfp8 indexer wk / weights_proj must reach the fused bf16 wk_weights_proj dequantized with its UE8M0 scales."""

import unittest
from types import SimpleNamespace

import torch

from sglang.srt.layers.quantization.mxfp8_block_convert import (
dequant_mxfp8_2d_to_bf16,
)
from sglang.srt.models.deepseek_common.deepseek_weight_loader import (
_load_fused_indexer_wk,
)
from sglang.test.ci.ci_register import register_cpu_ci
from sglang.test.test_utils import CustomTestCase

register_cpu_ci(est_time=1, suite="base-a-test-cpu")

HEAD_DIM, N_HEADS, HIDDEN = 128, 32, 256
PREFIX = "model.layers.0.self_attn.indexer"


class TestFusedIndexerMxfp8(CustomTestCase):
def test_mxfp8_shards_are_dequantized_with_their_ue8m0_scales(self):
"""The scales are biased exponents: used as multipliers, the indexer keys
are off by orders of magnitude until a bf16 update overwrites them."""
torch.manual_seed(0)
fused = torch.zeros(HEAD_DIM + N_HEADS, HIDDEN, dtype=torch.bfloat16)
params_dict = {f"{PREFIX}.wk_weights_proj.weight": fused}
quant_config = SimpleNamespace(weight_block_size=[1, 32])
pending = {}
expected = {}
for shard, rows in (("wk", HEAD_DIM), ("weights_proj", N_HEADS)):
weight = torch.randn(rows, HIDDEN).to(torch.float8_e4m3fn)
scale = torch.randint(120, 130, (rows, HIDDEN // 32), dtype=torch.uint8)
expected[shard] = dequant_mxfp8_2d_to_bf16(weight, scale)
for name, tensor in (("weight", weight), ("weight_scale_inv", scale)):
self.assertTrue(
_load_fused_indexer_wk(
f"{PREFIX}.{shard}.{name}",
tensor,
params_dict,
pending,
quant_config,
)
)

self.assertTrue(torch.equal(fused[:HEAD_DIM], expected["wk"]))
self.assertTrue(torch.equal(fused[-N_HEADS:], expected["weights_proj"]))


if __name__ == "__main__":
unittest.main()
Loading