From f94c48c6c4f28be6edfd8d7a33bc888b56cbcbfd Mon Sep 17 00:00:00 2001 From: Marvin Tsai <62472426+mqhc2020@users.noreply.github.com> Date: Tue, 2 Jun 2026 11:28:01 +0000 Subject: [PATCH 1/4] Add check with deep_compare --- .../srt/layers/quantization/quark/quark.py | 40 +++++++++++++++++++ python/sglang/srt/models/qwen2_moe.py | 15 ++----- python/sglang/srt/models/qwen3_5.py | 17 +++++++- 3 files changed, 60 insertions(+), 12 deletions(-) diff --git a/python/sglang/srt/layers/quantization/quark/quark.py b/python/sglang/srt/layers/quantization/quark/quark.py index 4b0f673a096c..4104df5934bb 100644 --- a/python/sglang/srt/layers/quantization/quark/quark.py +++ b/python/sglang/srt/layers/quantization/quark/quark.py @@ -35,6 +35,18 @@ logger = logging.getLogger(__name__) +_MOE_SHARED_EXPERT_QUANT_LAYER0_BASES: tuple[str, ...] = ( + "model.layers.0", + "model.language_model.layers.0", +) + +_SHARED_EXPERT_BODY_PROJ_SUFFIXES: tuple[str, ...] = ( + "gate_proj", + "up_proj", + "gate_up_proj", + "down_proj", +) + class QuarkConfig(QuantizationConfig): @@ -392,6 +404,34 @@ def get_moe_scheme( def get_scaled_act_names(self) -> List[str]: return [] + def can_fuse_shared_expert(self) -> bool: + if any( + "shared_expert" in layer + and "shared_expert_gate" not in layer + and not layer.startswith("mtp.") + for layer in self.exclude_layers + ): + return False + + layer_quant_config = self.quant_config.get("layer_quant_config") or {} + if not layer_quant_config: + return True + + lookup_stub = torch.nn.Module() + try: + for base in _MOE_SHARED_EXPERT_QUANT_LAYER0_BASES: + moe_name = f"{base}.mlp.experts" + moe_cfg = self._find_matched_config(moe_name, lookup_stub) + for suffix in _SHARED_EXPERT_BODY_PROJ_SUFFIXES: + shared_name = f"{base}.mlp.shared_expert.{suffix}" + shared_cfg = self._find_matched_config(shared_name, lookup_stub) + if not deep_compare(moe_cfg, shared_cfg): + return False + except ValueError: + return False + + return True + class QuarkLinearMethod(LinearMethodBase): diff --git a/python/sglang/srt/models/qwen2_moe.py b/python/sglang/srt/models/qwen2_moe.py index 305155e13e1f..b0f9ade7a4d4 100644 --- a/python/sglang/srt/models/qwen2_moe.py +++ b/python/sglang/srt/models/qwen2_moe.py @@ -124,9 +124,10 @@ def can_fuse_shared_expert( config: PretrainedConfig, quant_config: Optional[QuantizationConfig], ) -> bool: - """Whether the shared expert may be fused as an extra MoE expert (Qwen3.5 + Aiter). + """Whether the shared expert may be fused as an extra MoE expert (Qwen MoE + Aiter). Caller must still gate on ``support_shared_expert_fusion`` and ``_use_aiter``. + Optional ``quant_config.can_fuse_shared_expert()`` (e.g. Quark) may forbid fusion. """ if ( get_global_server_args().disable_shared_experts_fusion is True @@ -136,17 +137,9 @@ def can_fuse_shared_expert( ): return False - # If the shared expert is excluded from quantization (stored as FP32 in the - # checkpoint), fusing it into the quantized MoE weight tensor requires online - # quantization which is not supported. Disable fusion in this case. if quant_config is not None: - exclude_layers = getattr(quant_config, "exclude_layers", []) - if any( - "shared_expert" in layer - and "shared_expert_gate" not in layer - and not layer.startswith("mtp.") - for layer in exclude_layers - ): + can_fuse_fn = getattr(quant_config, "can_fuse_shared_expert", None) + if can_fuse_fn is not None and not can_fuse_fn(): return False return True diff --git a/python/sglang/srt/models/qwen3_5.py b/python/sglang/srt/models/qwen3_5.py index 9a95dd422098..6f55b8122603 100644 --- a/python/sglang/srt/models/qwen3_5.py +++ b/python/sglang/srt/models/qwen3_5.py @@ -75,7 +75,11 @@ default_weight_loader, sharded_weight_loader, ) -from sglang.srt.models.qwen2_moe import Qwen2MoeMLP, Qwen2MoeSparseMoeBlock +from sglang.srt.models.qwen2_moe import ( + Qwen2MoeMLP, + Qwen2MoeSparseMoeBlock, + can_fuse_shared_expert, +) # Models from sglang.srt.models.qwen3_vl import Qwen3VLForConditionalGeneration @@ -1051,6 +1055,16 @@ def get_hidden_dim(self, module_name: str, layer_idx: int): f"get_hidden_dim not implemented for {module_name}" ) + def _maybe_autodisable_shared_experts_fusion(self, config, quant_config): + server_args = get_global_server_args() + if ( + config.model_type == "qwen3_5_moe_text" + and not server_args.disable_shared_experts_fusion + and not can_fuse_shared_expert(config, quant_config) + ): + server_args.disable_shared_experts_fusion = True + logger.info("Qwen3.5: shared-expert fusion is not possible ... ") + def __init__( self, config: Qwen3_5TextConfig, @@ -1062,6 +1076,7 @@ def __init__( self.config = config self.hidden_size = config.hidden_size self.pp_group = get_pp_group() + self._maybe_autodisable_shared_experts_fusion(config, quant_config) alt_stream = torch.cuda.Stream() if _is_cuda else None From 24a8a79fc28f9516826b8a44e8895767975f4fd6 Mon Sep 17 00:00:00 2001 From: Marvin Tsai <62472426+mqhc2020@users.noreply.github.com> Date: Tue, 2 Jun 2026 15:03:02 +0000 Subject: [PATCH 2/4] fix lint --- python/sglang/srt/models/qwen3_5.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/python/sglang/srt/models/qwen3_5.py b/python/sglang/srt/models/qwen3_5.py index 6f55b8122603..66511a04b06a 100644 --- a/python/sglang/srt/models/qwen3_5.py +++ b/python/sglang/srt/models/qwen3_5.py @@ -1062,8 +1062,8 @@ def _maybe_autodisable_shared_experts_fusion(self, config, quant_config): and not server_args.disable_shared_experts_fusion and not can_fuse_shared_expert(config, quant_config) ): - server_args.disable_shared_experts_fusion = True - logger.info("Qwen3.5: shared-expert fusion is not possible ... ") + server_args.disable_shared_experts_fusion = True + logger.info("Qwen3.5: shared-expert fusion is not possible ... ") def __init__( self, From 95f52a172072ddb44c3d2cc0c9a1b66f3d2c7586 Mon Sep 17 00:00:00 2001 From: Marvin Tsai <62472426+mqhc2020@users.noreply.github.com> Date: Fri, 5 Jun 2026 11:14:36 +0000 Subject: [PATCH 3/4] add comment --- python/sglang/srt/models/qwen2_moe.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/python/sglang/srt/models/qwen2_moe.py b/python/sglang/srt/models/qwen2_moe.py index b0f9ade7a4d4..d07bbf865e0c 100644 --- a/python/sglang/srt/models/qwen2_moe.py +++ b/python/sglang/srt/models/qwen2_moe.py @@ -138,6 +138,8 @@ def can_fuse_shared_expert( return False if quant_config is not None: + # Other backends than quark do not exclude the shared expert here, so they + # intentionally fall through and remain fusable can_fuse_fn = getattr(quant_config, "can_fuse_shared_expert", None) if can_fuse_fn is not None and not can_fuse_fn(): return False From b6167a4314062534b9a83f2ad5e30c1cb1bd8d0a Mon Sep 17 00:00:00 2001 From: Marvin Tsai <62472426+mqhc2020@users.noreply.github.com> Date: Mon, 8 Jun 2026 13:50:56 +0000 Subject: [PATCH 4/4] Add more comments and isolate hip code --- python/sglang/srt/layers/quantization/quark/quark.py | 4 ++++ python/sglang/srt/models/qwen2_moe.py | 5 +++-- python/sglang/srt/models/qwen3_5.py | 11 +++++++++-- 3 files changed, 16 insertions(+), 4 deletions(-) diff --git a/python/sglang/srt/layers/quantization/quark/quark.py b/python/sglang/srt/layers/quantization/quark/quark.py index 4104df5934bb..5bb8196a6bb9 100644 --- a/python/sglang/srt/layers/quantization/quark/quark.py +++ b/python/sglang/srt/layers/quantization/quark/quark.py @@ -405,6 +405,7 @@ def get_scaled_act_names(self) -> List[str]: return [] def can_fuse_shared_expert(self) -> bool: + # Shared-expert body excluded from quant; the gate must not veto fusion. if any( "shared_expert" in layer and "shared_expert_gate" not in layer @@ -413,10 +414,13 @@ def can_fuse_shared_expert(self) -> bool: ): return False + # No per-layer config -> uniform spec, nothing to compare. layer_quant_config = self.quant_config.get("layer_quant_config") or {} if not layer_quant_config: return True + # Compare routed vs shared specs at layer 0 (stub module needed by + # _find_matched_config; an unmatched name -> ValueError -> cannot fuse). lookup_stub = torch.nn.Module() try: for base in _MOE_SHARED_EXPERT_QUANT_LAYER0_BASES: diff --git a/python/sglang/srt/models/qwen2_moe.py b/python/sglang/srt/models/qwen2_moe.py index d07bbf865e0c..cbfb23cb8c65 100644 --- a/python/sglang/srt/models/qwen2_moe.py +++ b/python/sglang/srt/models/qwen2_moe.py @@ -141,8 +141,9 @@ def can_fuse_shared_expert( # Other backends than quark do not exclude the shared expert here, so they # intentionally fall through and remain fusable can_fuse_fn = getattr(quant_config, "can_fuse_shared_expert", None) - if can_fuse_fn is not None and not can_fuse_fn(): - return False + if can_fuse_fn is not None: + if not can_fuse_fn(): + return False return True diff --git a/python/sglang/srt/models/qwen3_5.py b/python/sglang/srt/models/qwen3_5.py index 66511a04b06a..ecd4b2009e79 100644 --- a/python/sglang/srt/models/qwen3_5.py +++ b/python/sglang/srt/models/qwen3_5.py @@ -1056,6 +1056,8 @@ def get_hidden_dim(self, module_name: str, layer_idx: int): ) def _maybe_autodisable_shared_experts_fusion(self, config, quant_config): + # Auto-disable fusion when the checkpoint can't fuse (e.g. MXFP4 Qwen3.5) + # so the model still gets the #25885 multi-streaming path. ROCm-only. server_args = get_global_server_args() if ( config.model_type == "qwen3_5_moe_text" @@ -1063,7 +1065,10 @@ def _maybe_autodisable_shared_experts_fusion(self, config, quant_config): and not can_fuse_shared_expert(config, quant_config) ): server_args.disable_shared_experts_fusion = True - logger.info("Qwen3.5: shared-expert fusion is not possible ... ") + logger.info( + "Qwen3.5: shared-expert fusion not supported for this checkpoint; " + "auto-disabling (multi-streaming #25885 still applies)." + ) def __init__( self, @@ -1076,7 +1081,9 @@ def __init__( self.config = config self.hidden_size = config.hidden_size self.pp_group = get_pp_group() - self._maybe_autodisable_shared_experts_fusion(config, quant_config) + + if _is_hip: + self._maybe_autodisable_shared_experts_fusion(config, quant_config) alt_stream = torch.cuda.Stream() if _is_cuda else None