diff --git a/python/sglang/srt/layers/moe/fused_moe_triton/layer.py b/python/sglang/srt/layers/moe/fused_moe_triton/layer.py index 40d89d8023f5..64e46eac6858 100644 --- a/python/sglang/srt/layers/moe/fused_moe_triton/layer.py +++ b/python/sglang/srt/layers/moe/fused_moe_triton/layer.py @@ -1098,22 +1098,34 @@ def _maybe_load_fp8_shared_expert_as_fp4( ): return False + # Quark scales are raw uint8 E8M0 bytes. + is_quark = self.quant_config.get_name() == "quark" + if ( + is_quark + and "weight_scale" in weight_name + and loaded_weight.dtype == torch.uint8 + ): + loaded_weight = loaded_weight.view(torch.float8_e8m0fnu) is_weight = ( "weight" in weight_name and "scale" not in weight_name and loaded_weight.dtype == torch.float8_e4m3fn ) - is_scale = "weight_scale_inv" in weight_name and loaded_weight.dtype in ( - torch.float8_e8m0fnu, - torch.float32, - ) + # Quark names this weight_scale. Other formats use weight_scale_inv. + is_scale = ( + "weight_scale_inv" in weight_name + or (is_quark and "weight_scale" in weight_name) + ) and loaded_weight.dtype in (torch.float8_e8m0fnu, torch.float32) if not is_weight and not is_scale: return False - weight_param = self.w2_weight if shard_id == "w2" else self.w13_weight - scale_param = ( - self.w2_weight_scale_inv if shard_id == "w2" else self.w13_weight_scale_inv - ) + # Fall back to Quark's weight_scale name when weight_scale_inv is absent. + prefix = "w2" if shard_id == "w2" else "w13" + weight_param = getattr(self, f"{prefix}_weight") + scale_name = f"{prefix}_weight_scale_inv" + if is_quark and not hasattr(self, scale_name): + scale_name = f"{prefix}_weight_scale" + scale_param = getattr(self, scale_name) if param is not weight_param and param is not scale_param: return False @@ -1140,13 +1152,22 @@ def _maybe_load_fp8_shared_expert_as_fp4( weight_block_size = getattr(self.quant_config, "weight_block_size", None) if weight_block_size is None: - raise ValueError( - "Loading FP8 shared expert weights into FP4 fused MoE weights " - "requires block-FP8 weight_block_size." - ) + # Quark omits this. MXFP4 groups are 32. + if is_quark: + weight_block_size = [1, 32] + else: + raise ValueError( + "Loading FP8 shared expert weights into FP4 fused MoE weights " + "requires block-FP8 weight_block_size." + ) fp4_weight, fp4_scale = quantize_block_fp8_weight_to_mxfp4( fp8_weight, fp8_scale, weight_block_size ) + # Packed Quark buffers are uint8. + if is_quark and weight_param.dtype == torch.uint8: + fp4_weight = fp4_weight.view(torch.uint8) + if is_quark and scale_param.dtype == torch.uint8: + fp4_scale = fp4_scale.contiguous().view(torch.uint8) weight_data = weight_param.data[expert_id] scale_data = scale_param.data[expert_id] diff --git a/python/sglang/srt/layers/quantization/fp8.py b/python/sglang/srt/layers/quantization/fp8.py index ca511647bbab..c115212e0d32 100644 --- a/python/sglang/srt/layers/quantization/fp8.py +++ b/python/sglang/srt/layers/quantization/fp8.py @@ -272,6 +272,7 @@ def __init__( is_fp4_experts: bool = False, kv_cache_quant_algo: Optional[str] = None, scale_fmt: Optional[str] = None, + weight_scale_is_ue8m0: bool = False, ) -> None: super().__init__() # DSV4 mxfp4-packed (True) vs converted FP8 (False); injected by @@ -299,6 +300,8 @@ def __init__( self.kv_cache_quant_algo = kv_cache_quant_algo # "ue8m0" checkpoints quantize activations with power-of-two scales. self.scale_fmt = scale_fmt + # When set, the block-FP8 scale is a raw E8M0 byte. + self.weight_scale_is_ue8m0 = weight_scale_is_ue8m0 if weight_block_size is not None: if not is_checkpoint_fp8_serialized: raise ValueError( @@ -773,6 +776,10 @@ def process_weights_after_loading_block_quant(self, layer: Module) -> None: if self.use_npu_block_fp8: # G-B consumes checkpoint-layout E4M3 weights and FP32 scales. return + if getattr(self.quant_config, "weight_scale_is_ue8m0", False): + # Raw E8M0 byte -> 2^(byte - 127). + scale = layer.weight_scale_inv.data + layer.weight_scale_inv.data = torch.exp2(scale - 127) use_deepgemm_runner = ( getattr(self.w8a8_block_fp8_linear, "func", self.w8a8_block_fp8_linear) is deepgemm_w8a8_block_fp8_linear_with_fallback diff --git a/python/sglang/srt/layers/quantization/quark/quark.py b/python/sglang/srt/layers/quantization/quark/quark.py index ce797984d475..a4f894b752b8 100644 --- a/python/sglang/srt/layers/quantization/quark/quark.py +++ b/python/sglang/srt/layers/quantization/quark/quark.py @@ -77,6 +77,18 @@ def _parse_nvfp4_excludes(hf_quant_config: Dict[str, Any]) -> List[str]: return [p if p.startswith("re:") else "re:" + fnmatch.translate(p) for p in pats] +def _minimax_quark_routed_experts_are_fp4( + model_type: Optional[str], quant_config: dict[str, Any] +) -> bool: + """MiniMax Quark stores routed experts as global FP4.""" + if model_type != "minimax_m3_vl": + return False + weight = (quant_config.get("global_quant_config") or {}).get("weight") or {} + if not isinstance(weight, dict): + return False + return str(weight.get("dtype", "")).lower() == "fp4" + + def _detect_nvfp4_source(config: Dict[str, Any]) -> Optional["Nvfp4SourceConfig"]: """Return an Nvfp4SourceConfig if `config` (the checkpoint's quantization_config dict) describes a supported NVFP4 source, else None. @@ -364,6 +376,10 @@ def __init__( # Gates the online shared-expert MXFP4 conversion to the architectures # it is supported on; see _ONLINE_SHARED_EXPERT_MXFP4_MODEL_TYPES. self.model_type = getattr(hf_config, "model_type", None) + # MiniMax Quark marks routed experts via the global weight dtype. + self.is_fp4_experts = _minimax_quark_routed_experts_are_fp4( + self.model_type, quant_config + ) self.is_prequantized = is_prequantized self.dequantization_config = dequantization_config # Load-as-is FP8 config for excluded layers of a mixed-precision source @@ -454,6 +470,8 @@ def _get_block_fp8_config( activation_scheme="dynamic", weight_block_size=block_size, packed_modules_mapping=packed_modules_mapping, + # True when the scale is a raw E8M0 byte. + weight_scale_is_ue8m0=weight_config.get("scale_type") == "float8_e8m0fnu", ) def is_linear_unquantized(self, prefix: str) -> bool: diff --git a/python/sglang/srt/model_loader/loader.py b/python/sglang/srt/model_loader/loader.py index edda2a761822..d52622f7ecb3 100644 --- a/python/sglang/srt/model_loader/loader.py +++ b/python/sglang/srt/model_loader/loader.py @@ -250,6 +250,8 @@ def _get_quantization_config( ) elif quant_config.get_name() == "humming": quant_config.is_fp4_experts = model_config.is_fp4_experts + elif quant_config.get_name() == "quark" and model_config.is_fp4_experts: + quant_config.is_fp4_experts = True if not _is_npu: major, minor = get_device_capability()