diff --git a/python/sglang/srt/layers/quantization/fp8_utils.py b/python/sglang/srt/layers/quantization/fp8_utils.py index 8344cb9a220c..0ffee501442c 100755 --- a/python/sglang/srt/layers/quantization/fp8_utils.py +++ b/python/sglang/srt/layers/quantization/fp8_utils.py @@ -1937,24 +1937,24 @@ def _inverse_transform_scale_ue8m0_impl(sf_packed): assert len(sf_packed.shape) == 2, f"{sf_packed.shape=}" assert sf_packed.dtype == torch.int32 - mn_repeat_128, k_div_4 = sf_packed.shape - mn = mn_repeat_128 // block_size + # one row per weight row, so the last block is partial when the rows are not a multiple of 128 + num_rows, k_div_4 = sf_packed.shape + mn = -(-num_rows // block_size) k = k_div_4 * 4 # packed u8 -> fp32 - sf_u8 = sf_packed.contiguous().flatten().view(torch.uint8).view(mn_repeat_128, k) + sf_u8 = sf_packed.contiguous().flatten().view(torch.uint8).view(num_rows, k) sf_fp32 = (sf_u8.to(torch.int32) << 23).view(torch.float32) # remove repeat - sf_reshaped = sf_fp32.view(mn, block_size, k) - sf_unrepeated = sf_reshaped[:, 0:1, :] - if not torch.all(sf_unrepeated == sf_reshaped): + sf_unrepeated = sf_fp32[::block_size].contiguous() + sf_repeated = sf_unrepeated.repeat_interleave(block_size, dim=0)[:num_rows] + if not torch.all(sf_repeated == sf_fp32): from sglang.srt.debug_utils.dumper import get_tensor_info raise AssertionError( - f"sf_unrepeated != sf_reshaped ({get_tensor_info(sf_unrepeated)=} {get_tensor_info(sf_reshaped)=})" + f"sf_repeated != sf_fp32 ({get_tensor_info(sf_repeated)=} {get_tensor_info(sf_fp32)=})" ) - sf_unrepeated = sf_unrepeated.squeeze(1).contiguous() assert sf_unrepeated.shape == (mn, k) return sf_unrepeated diff --git a/python/sglang/srt/layers/quantization/modelopt_quant.py b/python/sglang/srt/layers/quantization/modelopt_quant.py index 806c52bc25b8..7daad0580d31 100755 --- a/python/sglang/srt/layers/quantization/modelopt_quant.py +++ b/python/sglang/srt/layers/quantization/modelopt_quant.py @@ -2574,6 +2574,9 @@ def create_weights( ) w2_input_scale._sglang_require_global_experts = True layer.register_parameter("w2_input_scale", w2_input_scale) + # calibrated activation scales, as the KV cache scales: no weight update carries them + w13_input_scale._skip_weight_check = True + w2_input_scale._skip_weight_check = True def _build_mega_moe_weights(self, layer: torch.nn.Module) -> None: # Activations are quantized per token at dispatch, so w13_input_scale diff --git a/python/sglang/srt/model_executor/model_runner_components/spec_aux_hidden_state.py b/python/sglang/srt/model_executor/model_runner_components/spec_aux_hidden_state.py index 950761105fae..a5d38150828e 100644 --- a/python/sglang/srt/model_executor/model_runner_components/spec_aux_hidden_state.py +++ b/python/sglang/srt/model_executor/model_runner_components/spec_aux_hidden_state.py @@ -94,6 +94,11 @@ def _resolve_eagle_aux_hidden_state( is_draft_model=True, ) num_nextn_predict_layers = draft_model_config.num_nextn_predict_layers + if num_nextn_predict_layers is None: + # Qwen3.5 names its MTP depth mtp_num_hidden_layers; models without MTP have neither + num_nextn_predict_layers = getattr( + draft_model_config.hf_text_config, "mtp_num_hidden_layers", None + ) if num_nextn_predict_layers is not None: config.eagle_draft_num_layers = int(num_nextn_predict_layers) elif get_spec().speculative_draft_model_path: diff --git a/python/sglang/srt/model_executor/model_runner_components/weight_updater.py b/python/sglang/srt/model_executor/model_runner_components/weight_updater.py index 267f2c7e665c..072c97212c68 100644 --- a/python/sglang/srt/model_executor/model_runner_components/weight_updater.py +++ b/python/sglang/srt/model_executor/model_runner_components/weight_updater.py @@ -12,6 +12,7 @@ DefaultModelLoader, get_model_loader, post_load_weights, + refresh_gemma_weights, ) from sglang.srt.model_loader.utils import set_default_torch_dtype from sglang.srt.model_loader.weight_utils import default_weight_loader @@ -306,6 +307,8 @@ def begin_weight_update(self: WeightUpdater) -> None: def end_weight_update(self: WeightUpdater, *, run_post_load: bool) -> None: if run_post_load: post_load_weights(self.get_model()) + else: + refresh_gemma_weights(self.get_model()) DefaultModelLoader.postprocess_weights( self.get_model(), torch.device(self.device) ) diff --git a/python/sglang/srt/model_loader/loader.py b/python/sglang/srt/model_loader/loader.py index 4be00fa62fe0..ae8266aede23 100644 --- a/python/sglang/srt/model_loader/loader.py +++ b/python/sglang/srt/model_loader/loader.py @@ -326,7 +326,11 @@ def post_load_weights(model: nn.Module) -> None: # `is_nextn=True`, so the loader doesn't need to know. if hasattr(model, "post_load_weights"): model.post_load_weights() - # its weight loader derives the buffer, and these loaders bypass it + refresh_gemma_weights(model) + + +def refresh_gemma_weights(model: nn.Module) -> None: + # its weight loader derives the buffer, so writes that bypass the loader leave it stale for module in model.modules(): if isinstance(module, GemmaRMSNorm): module.refresh_gemma_weight() diff --git a/python/sglang/srt/models/deepseek_common/deepseek_weight_loader.py b/python/sglang/srt/models/deepseek_common/deepseek_weight_loader.py index 774f325b7c75..3b4607b8bfd0 100644 --- a/python/sglang/srt/models/deepseek_common/deepseek_weight_loader.py +++ b/python/sglang/srt/models/deepseek_common/deepseek_weight_loader.py @@ -39,6 +39,7 @@ from sglang.srt.layers.quantization.int8_utils import ( block_dequant as int8_block_dequant, ) +from sglang.srt.layers.quantization.mxfp8_block_convert import dequant_mxfp8_2d_to_bf16 from sglang.srt.layers.utils import get_layer_id from sglang.srt.model_loader.utils import ( maybe_executor_submit, @@ -86,6 +87,19 @@ def _get_indexer_weight_block_size( return [128, 128] +def _dequant_fused_indexer_shard( + weight: torch.Tensor, + scale: torch.Tensor, + quant_config: Optional[QuantizationConfig], +) -> torch.Tensor: + # mxfp8 scales are UE8M0 exponents stored as uint8, not fp32 multipliers + if scale.dtype == torch.uint8: + return dequant_mxfp8_2d_to_bf16(weight, scale) + return block_quant_dequant( + weight, scale, _get_indexer_weight_block_size(quant_config), torch.bfloat16 + ) + + def _load_fused_indexer_wk( name: str, loaded_weight: torch.Tensor, @@ -119,9 +133,8 @@ def _load_fused_indexer_wk( ) if "weight" in entry and "scale" in entry: pending.pop(fused_name + ".weights_proj") - block_size = _get_indexer_weight_block_size(quant_config) - weights_bf16 = block_quant_dequant( - entry["weight"], entry["scale"], block_size, torch.bfloat16 + weights_bf16 = _dequant_fused_indexer_shard( + entry["weight"], entry["scale"], quant_config ) fused_param.data[-weights_bf16.shape[0] :].copy_(weights_bf16) return True @@ -139,9 +152,8 @@ def _load_fused_indexer_wk( ) if "weight" in entry and "scale" in entry: pending.pop(fused_name) - block_size = _get_indexer_weight_block_size(quant_config) - wk_bf16 = block_quant_dequant( - entry["weight"], entry["scale"], block_size, torch.bfloat16 + wk_bf16 = _dequant_fused_indexer_shard( + entry["weight"], entry["scale"], quant_config ) fused_param.data[: wk_bf16.shape[0]].copy_(wk_bf16) return True diff --git a/test/registered/unit/layers/quantization/test_fp8_utils.py b/test/registered/unit/layers/quantization/test_fp8_utils.py index d0ed39ca3a1c..10626babd195 100644 --- a/test/registered/unit/layers/quantization/test_fp8_utils.py +++ b/test/registered/unit/layers/quantization/test_fp8_utils.py @@ -82,13 +82,18 @@ def is_deep_gemm(self): class TestInverseTransformScaleUe8m0(CustomTestCase): def test_round_trip(self): + for shape in ( + # DeepSeek V3 kv_b_proj + (32768, 512), + # GLM-5 fused q_a + kv_a: 2048 + 576 rows, the last block partial + (2624, 6144), + ): + with self.subTest(shape=shape): + self._check_round_trip(shape) + + def _check_round_trip(self, shape): for _ in range(100): - weight_bf16 = torch.randn( - # DeepSeek V3 kv_b_proj - (32768, 512), - dtype=torch.bfloat16, - device="cuda", - ) + weight_bf16 = torch.randn(shape, dtype=torch.bfloat16, device="cuda") weight_block_size = [128, 128] diff --git a/test/registered/unit/layers/quantization/test_modelopt_nvfp4_moe_weight_check.py b/test/registered/unit/layers/quantization/test_modelopt_nvfp4_moe_weight_check.py new file mode 100644 index 000000000000..df3d941fb93b --- /dev/null +++ b/test/registered/unit/layers/quantization/test_modelopt_nvfp4_moe_weight_check.py @@ -0,0 +1,81 @@ +"""The weight checker must leave the NVFP4 MoE activation input scales alone: no weight update carries them, so +a reset before an update leaves them random and the comparison after it fails whatever the update wrote. + +The platform check is stubbed so the layer is built on CPU, as in test_modelopt_nvfp4_moe_dispatch.py. +""" + +from sglang.test.ci.ci_register import register_cpu_ci + +register_cpu_ci(est_time=6, suite="base-a-test-cpu") + +import unittest +from types import SimpleNamespace +from unittest import mock + +import torch + +# Import modelopt_quant first (see test_modelopt_nvfp4_moe_scales.py for the circular-import reason). +# isort: off +from sglang.srt.layers.quantization import modelopt_quant +from sglang.srt.layers.quantization.modelopt_quant import ModelOptNvFp4FusedMoEMethod + +# isort: on +from sglang.srt.layers.moe.utils import MoeRunnerBackend +from sglang.srt.utils.weight_checker import _is_skip_weight_check +from sglang.test.test_utils import CustomTestCase + +NUM_EXPERTS = 8 +HIDDEN = 64 +INTERMEDIATE = 32 + + +def _nvfp4_moe_layer() -> torch.nn.Module: + with ( + mock.patch.object( + modelopt_quant, + "get_moe_runner_backend", + return_value=MoeRunnerBackend.FLASHINFER_TRTLLM_ROUTED, + ), + mock.patch.object( + modelopt_quant, + "get_platform", + return_value=SimpleNamespace(is_blackwell=True), + ), + mock.patch.object(modelopt_quant, "is_cuda", return_value=False), + ): + method = ModelOptNvFp4FusedMoEMethod( + SimpleNamespace( + group_size=16, + is_checkpoint_nvfp4_serialized=True, + use_per_token_activation=False, + get_name=lambda: "modelopt_fp4", + ) + ) + layer = torch.nn.Module() + layer.num_experts = NUM_EXPERTS + layer.num_local_experts = NUM_EXPERTS + layer.moe_ep_rank = 0 + layer.moe_runner_config = SimpleNamespace(is_gated=True) + method.create_weights( + layer, + num_experts=NUM_EXPERTS, + hidden_size=HIDDEN, + intermediate_size_per_partition=INTERMEDIATE, + params_dtype=torch.bfloat16, + weight_loader=lambda *args, **kwargs: None, + ) + return layer + + +class TestNvFp4MoeWeightCheck(CustomTestCase): + def test_the_checker_skips_the_input_scales_and_checks_the_weight_scales(self): + params = dict(_nvfp4_moe_layer().named_parameters()) + + for name in ("w13_input_scale", "w2_input_scale"): + self.assertTrue(_is_skip_weight_check(name, params[name]), name) + for name in ("w13_weight", "w13_weight_scale", "w13_weight_scale_2"): + self.assertFalse(_is_skip_weight_check(name, params[name]), name) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/registered/unit/model_executor/model_runner_components/test_spec_aux_hidden_state.py b/test/registered/unit/model_executor/model_runner_components/test_spec_aux_hidden_state.py index a48604188f99..0b28912dd11e 100644 --- a/test/registered/unit/model_executor/model_runner_components/test_spec_aux_hidden_state.py +++ b/test/registered/unit/model_executor/model_runner_components/test_spec_aux_hidden_state.py @@ -3,9 +3,13 @@ import pytest +from sglang.srt.model_executor.model_runner_components import spec_aux_hidden_state from sglang.srt.model_executor.model_runner_components.spec_aux_hidden_state import ( + SpecAuxHiddenStateConfig, _map_muse_target_layer_ids, + _resolve_eagle_aux_hidden_state, ) +from sglang.srt.speculative.spec_info import SpeculativeAlgorithm from sglang.test.ci.ci_register import register_cpu_ci register_cpu_ci(est_time=11, suite="base-a-test-cpu") @@ -34,5 +38,34 @@ def test_muse_target_layer_id_mapping(target_model_type, draft_architecture, exp ) +def test_an_mtp_depth_named_mtp_num_hidden_layers_counts_as_the_eagle_draft( + monkeypatch, +): + """Qwen3.5 names its MTP depth mtp_num_hidden_layers. Read as no draft, the target's + KV budget leaves the draft's pool, of the target's token count, outside + mem_fraction_static.""" + monkeypatch.setattr( + spec_aux_hidden_state, + "get_spec", + lambda: SimpleNamespace(speculative_draft_model_path=None), + ) + config = SpecAuxHiddenStateConfig() + + _resolve_eagle_aux_hidden_state( + config=config, + server_args=None, + model_config=SimpleNamespace( + num_nextn_predict_layers=None, + hf_text_config=SimpleNamespace(mtp_num_hidden_layers=1), + is_hybrid_swa=False, + is_deepseek_v4_arch=False, + ), + spec_algorithm=SpeculativeAlgorithm.EAGLE, + is_draft_worker=False, + ) + + assert config.eagle_draft_num_layers == 1 + + if __name__ == "__main__": sys.exit(pytest.main([__file__, "-v"])) diff --git a/test/registered/unit/model_executor/model_runner_components/test_weight_updater_published_storage.py b/test/registered/unit/model_executor/model_runner_components/test_weight_updater_published_storage.py index b2de83768ef8..4cb81a6fb3b4 100644 --- a/test/registered/unit/model_executor/model_runner_components/test_weight_updater_published_storage.py +++ b/test/registered/unit/model_executor/model_runner_components/test_weight_updater_published_storage.py @@ -1,4 +1,5 @@ -"""A weight-update session must leave every parameter that peers write by address where it was published.""" +"""A weight-update session must leave every parameter that peers write by address where it was published, and +rebuild what the loaders derive from the weights it wrote.""" import unittest from contextlib import contextmanager @@ -8,6 +9,7 @@ import torch import sglang.srt.model_executor.model_runner_components.weight_updater as weight_updater_mod +from sglang.srt.layers.layernorm import GemmaRMSNorm from sglang.srt.model_executor.model_runner_components.weight_updater import ( WeightUpdater, ) @@ -96,5 +98,22 @@ def test_a_rank_that_published_nothing_is_not_checked(self): weight_updater.end_weight_update(run_post_load=False) +class TestDerivedState(CustomTestCase): + def test_gemma_weight_follows_a_norm_weight_loaded_without_its_loader(self): + """`direct` and custom loads write params without GemmaRMSNorm's loader yet mark the session loaded, so + fused allreduce kernels would read the old `gemma_weight`.""" + model = torch.nn.Module() + model.norm = GemmaRMSNorm(4) + weight_updater = _weight_updater(model, published_weight_info=None) + new_weight = torch.tensor([0.5, -0.25, 0.0, 2.0]) + + with _postprocess(lambda model, device: None): + weight_updater.begin_weight_update() + model.norm.weight.data.copy_(new_weight) + weight_updater.end_weight_update(run_post_load=False) + + torch.testing.assert_close(model.norm.gemma_weight, new_weight + 1) + + if __name__ == "__main__": unittest.main() diff --git a/test/registered/unit/models/test_deepseek_fused_indexer_mxfp8.py b/test/registered/unit/models/test_deepseek_fused_indexer_mxfp8.py new file mode 100644 index 000000000000..860314fd1f57 --- /dev/null +++ b/test/registered/unit/models/test_deepseek_fused_indexer_mxfp8.py @@ -0,0 +1,53 @@ +"""An mxfp8 indexer wk / weights_proj must reach the fused bf16 wk_weights_proj dequantized with its UE8M0 scales.""" + +import unittest +from types import SimpleNamespace + +import torch + +from sglang.srt.layers.quantization.mxfp8_block_convert import ( + dequant_mxfp8_2d_to_bf16, +) +from sglang.srt.models.deepseek_common.deepseek_weight_loader import ( + _load_fused_indexer_wk, +) +from sglang.test.ci.ci_register import register_cpu_ci +from sglang.test.test_utils import CustomTestCase + +register_cpu_ci(est_time=1, suite="base-a-test-cpu") + +HEAD_DIM, N_HEADS, HIDDEN = 128, 32, 256 +PREFIX = "model.layers.0.self_attn.indexer" + + +class TestFusedIndexerMxfp8(CustomTestCase): + def test_mxfp8_shards_are_dequantized_with_their_ue8m0_scales(self): + """The scales are biased exponents: used as multipliers, the indexer keys + are off by orders of magnitude until a bf16 update overwrites them.""" + torch.manual_seed(0) + fused = torch.zeros(HEAD_DIM + N_HEADS, HIDDEN, dtype=torch.bfloat16) + params_dict = {f"{PREFIX}.wk_weights_proj.weight": fused} + quant_config = SimpleNamespace(weight_block_size=[1, 32]) + pending = {} + expected = {} + for shard, rows in (("wk", HEAD_DIM), ("weights_proj", N_HEADS)): + weight = torch.randn(rows, HIDDEN).to(torch.float8_e4m3fn) + scale = torch.randint(120, 130, (rows, HIDDEN // 32), dtype=torch.uint8) + expected[shard] = dequant_mxfp8_2d_to_bf16(weight, scale) + for name, tensor in (("weight", weight), ("weight_scale_inv", scale)): + self.assertTrue( + _load_fused_indexer_wk( + f"{PREFIX}.{shard}.{name}", + tensor, + params_dict, + pending, + quant_config, + ) + ) + + self.assertTrue(torch.equal(fused[:HEAD_DIM], expected["wk"])) + self.assertTrue(torch.equal(fused[-N_HEADS:], expected["weights_proj"])) + + +if __name__ == "__main__": + unittest.main()