Fix/compressed tensors nvfp4 hopper

python/sglang/srt/layers/quantization/compressed_tensors/README.md

@@ -2,5 +2,10 @@
 
 To support compressed_tensors format quantization models, we adapted https://github.com/vllm-project/vllm/tree/main/vllm/model_executor/layers/quantization/compressed_tensors into SGLang.
 
-
-For practical purposes, we have only applied the compressed_tensors format of `w8a8_fp8`. If you have requirements for other formats, you can submit an issue through this [link](https://github.com/sgl-project/sglang/issues).
+Several formats are supported today, for both linear and MoE layers -- fp8 and
+int8 w8a8, w8a16 fp8, int4 `pack-quantized`, and `nvfp4-pack-quantized` (native
+W4A4 on Blackwell, weight-only FP4 Marlin below it). See `schemes/` for the full
+set. `mxfp4-pack-quantized` linears are not supported yet: the dense FP4 Marlin
+kernel is only instantiated for group_size 16 and cannot decode E8M0 scales. If
+you have requirements for other formats, you can submit an issue through this
+[link](https://github.com/sgl-project/sglang/issues).

python/sglang/srt/layers/quantization/compressed_tensors/compressed_tensors.py

@@ -44,6 +44,8 @@
     CompressedTensorsMxInt4MoE,
     CompressedTensorsW4A4Fp4,
     CompressedTensorsW4A4Nvfp4MoE,
+    CompressedTensorsW4A16Fp4,
+    CompressedTensorsW4A16Nvfp4MoE,
     CompressedTensorsW4AFP8MoE,
     CompressedTensorsW8A8Fp8,
     CompressedTensorsW8A8Fp8MoE,
@@ -477,6 +479,12 @@ def _is_wint4abf16(self, weight_quant: BaseModel, input_quant: BaseModel) -> boo
     def _is_static_tensor_w8a8(
         self, weight_quant: BaseModel, input_quant: BaseModel
     ) -> bool:
+        # A weight-only checkpoint has no input_activations at all; without
+        # this guard the num_bits read below raises AttributeError instead of
+        # falling through to the weight-only branches.
+        if weight_quant is None or input_quant is None:
+            return False
+
         is_8_bits = weight_quant.num_bits == input_quant.num_bits == 8
         weight_strategy = (
             weight_quant.strategy == QuantizationStrategy.TENSOR.value
@@ -495,6 +503,12 @@ def _is_static_tensor_w8a8(
     def _is_dynamic_token_w8a8(
         self, weight_quant: BaseModel, input_quant: BaseModel
     ) -> bool:
+        # A weight-only checkpoint has no input_activations at all; without
+        # this guard the num_bits read below raises AttributeError instead of
+        # falling through to the weight-only branches.
+        if weight_quant is None or input_quant is None:
+            return False
+
         is_8_bits = weight_quant.num_bits == input_quant.num_bits == 8
         weight_strategy = (
             weight_quant.strategy == QuantizationStrategy.TENSOR.value
@@ -600,6 +614,21 @@ def _is_fp4a4_nvfp4(
             and is_symmetric
         )
 
+    def _is_fp4a16_nvfp4(
+        self, weight_quant: QuantizationArgs, input_quant: QuantizationArgs
+    ):
+        """Weight-only NVFP4: fp4 weights, unquantized activations."""
+        if weight_quant is None or input_quant is not None:
+            return False
+
+        return (
+            weight_quant.strategy == QuantizationStrategy.TENSOR_GROUP.value
+            and weight_quant.type == QuantizationType.FLOAT
+            and weight_quant.num_bits == 4
+            and weight_quant.group_size == 16
+            and weight_quant.symmetric
+        )
+
     def _is_wNa16_group_channel(
         self, weight_quant: BaseModel, input_quant: BaseModel
     ) -> bool:
@@ -609,11 +638,16 @@ def _is_wNa16_group_channel(
             or weight_quant.strategy == QuantizationStrategy.GROUP.value
         )
         is_static = not weight_quant.dynamic
+        # GROUP strategy alone does not imply integer weights: the float4
+        # weight-only formats (nvfp4a16, mxfp4a16) also match it, and routing
+        # them here reports a misleading W4A16Sparse24 error instead of falling
+        # through to the fp4 branches below.
+        is_int_type = weight_quant.type == QuantizationType.INT
 
         # Both symmetric and asymmetric weight quant are handled by
         # CompressedTensorsWNA16 via the Marlin kernel path; asymmetric
         # checkpoints carry a weight zero-point.
-        return is_channel_group and input_quant_none and is_static
+        return is_channel_group and input_quant_none and is_static and is_int_type
 
     def _is_wna16_triton_moe_supported(self, weight_quant: BaseModel) -> bool:
         return (
@@ -716,16 +750,42 @@ def _get_scheme_from_parts(
                 )
 
         if is_activation_quantization_format(quant_format):
+            if self._is_fp4a16_nvfp4(weight_quant, input_quant):
+                return CompressedTensorsW4A16Fp4()
+
             if self._is_fp4a4_nvfp4(weight_quant, input_quant):
                 is_fp4a4_nvfp4_supported = self._check_scheme_supported(
                     CompressedTensorsW4A4Fp4.get_min_capability(), error=False
                 )
                 if is_fp4a4_nvfp4_supported:
                     return CompressedTensorsW4A4Fp4()
-                else:
-                    raise NotImplementedError(
-                        "Current platform does not support w4a4 nvfp4 quantization."
-                    )
+                # note: input_global_scale will be present for w4a4 checkpoints;
+                # it is dropped post loading and activations stay unquantized.
+                logger.warning_once(
+                    "NVFP4 w4a4 requires compute capability >= 10.0; serving "
+                    "this checkpoint weight-only via FP4 Marlin instead. "
+                    "Activation quantization is ignored."
+                )
+                return CompressedTensorsW4A16Fp4(has_input_global_scale=True)
+
+            # Activation-quantized float4 that is not the nvfp4 layout: the
+            # branch above returns for every nvfp4 case, native or downgraded,
+            # so reaching here means the strategy or group_size is unsupported.
+            # Reported explicitly rather than falling through to the generic
+            # "no scheme found", which hides which field is at fault.
+            is_non_nvfp4_fp4a4 = (
+                weight_quant is not None
+                and input_quant is not None
+                and weight_quant.num_bits == 4
+                and weight_quant.type == QuantizationType.FLOAT
+            )
+            if is_non_nvfp4_fp4a4:
+                raise NotImplementedError(
+                    "Activation-quantized float4 is only supported in the NVFP4 "
+                    "layout (tensor_group, group_size 16); got "
+                    f"strategy={weight_quant.strategy}, "
+                    f"group_size={weight_quant.group_size}."
+                )
 
             if self._is_fp8_w8a8(weight_quant, input_quant):
                 if _is_xpu:
@@ -882,9 +942,23 @@ def get_moe_scheme(
                 ):
                     logger.info_once("Using NPUCompressedTensorsW4A16Int4DynamicMoE")
                     return NPUCompressedTensorsW4A16Int4DynamicMoE(self)
+        elif self._is_fp4a16_nvfp4(weight_quant, input_quant):
+            logger.info_once("Using CompressedTensorsW4A16Nvfp4MoE")
+            return CompressedTensorsW4A16Nvfp4MoE()
         elif self._is_fp4a4_nvfp4(weight_quant, input_quant):
-            logger.info_once("Using CompressedTensorsW4A4Nvfp4MoE")
-            return CompressedTensorsW4A4Nvfp4MoE()
+            if self._check_scheme_supported(
+                CompressedTensorsW4A4Nvfp4MoE.get_min_capability(), error=False
+            ):
+                logger.info_once("Using CompressedTensorsW4A4Nvfp4MoE")
+                return CompressedTensorsW4A4Nvfp4MoE()
+            # input_global_scale is present for w4a4 checkpoints; it is dropped
+            # post loading and activations stay unquantized.
+            logger.warning_once(
+                "NVFP4 w4a4 MoE requires compute capability >= 10.0; serving "
+                "this checkpoint weight-only via FP4 Marlin instead. "
+                "Activation quantization is ignored."
+            )
+            return CompressedTensorsW4A16Nvfp4MoE(has_input_global_scale=True)
         elif self._is_fp8_w8a8(weight_quant, input_quant):
             logger.info_once("Using CompressedTensorsW8A8Fp8MoE")
             return CompressedTensorsW8A8Fp8MoE(weight_quant, input_quant)

python/sglang/srt/layers/quantization/compressed_tensors/schemes/__init__.py

@@ -10,6 +10,8 @@
 from .compressed_tensors_w4a8_fp8_moe import CompressedTensorsW4AFP8MoE
 from .compressed_tensors_w4a8_int8_moe import NPUCompressedTensorsW4A8Int8DynamicMoE
 from .compressed_tensors_w4a8_mxfp4_moe import NPUCompressedTensorsW4A8mxfp4MoE
+from .compressed_tensors_w4a16_nvfp4 import CompressedTensorsW4A16Fp4
+from .compressed_tensors_w4a16_nvfp4_moe import CompressedTensorsW4A16Nvfp4MoE
 from .compressed_tensors_w8a8_fp8 import CompressedTensorsW8A8Fp8
 from .compressed_tensors_w8a8_fp8_moe import CompressedTensorsW8A8Fp8MoE
 from .compressed_tensors_w8a8_int8 import (
@@ -44,6 +46,8 @@
     "CompressedTensorsWNA16TritonMoE",
     "NPUCompressedTensorsW4A16Int4DynamicMoE",
     "WNA16_SUPPORTED_BITS",
+    "CompressedTensorsW4A16Fp4",
+    "CompressedTensorsW4A16Nvfp4MoE",
     "CompressedTensorsW4A4Fp4",
     "CompressedTensorsW4A4Nvfp4MoE",
     "NPUCompressedTensorsW4A8Int8DynamicMoE",