[diffusion] quant: Add flag for runtime quantization

python/sglang/multimodal_gen/runtime/loader/transformer_load_utils.py

@@ -485,8 +485,18 @@ def _resolve_quant_config(
         arch_config, "reverse_param_names_mapping", None
     )
 
-    quant_config = get_quant_config(hf_config, component_model_path)
+    quant_config = get_quant_config(
+        hf_config, component_model_path, server_args=server_args
+    )
     quant_config_name = _get_quant_config_name(quant_config)
+
+    if quant_config is not None or not server_args.transformer_weights_path:
+        return quant_config
+
+    quant_config = _resolve_quant_config_from_transformer_override(
+        server_args.transformer_weights_path
+    )
+
     inferred_nvfp4_config = None
     if quant_config is None or quant_config_name == "modelopt_fp4":
         fallback_group_size = None
@@ -498,13 +508,7 @@ def _resolve_quant_config(
             reverse_param_names_mapping_dict,
             fallback_group_size,
         )
-    quant_config = _merge_modelopt_fp4_configs(quant_config, inferred_nvfp4_config)
-    if quant_config is not None or not server_args.transformer_weights_path:
-        return quant_config
 
-    quant_config = _resolve_quant_config_from_transformer_override(
-        server_args.transformer_weights_path
-    )
     quant_config = _merge_modelopt_fp4_configs(quant_config, inferred_nvfp4_config)
     if quant_config is not None:
         return quant_config

python/sglang/multimodal_gen/runtime/server_args.py

@@ -180,6 +180,8 @@ class ServerArgs(DisaggArgsMixin):
 
     # path to pre-quantized transformer weights (single .safetensors or directory).
     transformer_weights_path: str | None = None
+    # runtime quantization method to apply
+    quantization: str | None = None
     # can restrict layers to adapt, e.g. ["q_proj"]
     # Will adapt only q, k, v, o by default.
     lora_target_modules: list[str] | None = None
@@ -991,6 +993,15 @@ def add_cli_args(parser: FlexibleArgumentParser) -> FlexibleArgumentParser:
             help="Disable autocast for denoising loop and vae decoding in pipeline sampling",
         )
 
+        parser.add_argument(
+            "--quantization",
+            type=str,
+            default=ServerArgs.quantization,
+            choices=["fp8"],
+            help="Apply runtime quantization to model weights. "
+            "Quantizes weights on-the-fly at load time, no pre-converted checkpoint needed.",
+        )
+
         # Nunchaku SVDQuant quantization parameters
         NunchakuSVDQuantArgs.add_cli_args(parser)
 

python/sglang/multimodal_gen/runtime/utils/quantization_utils.py

@@ -11,6 +11,8 @@
     QuantizationConfig,
     get_quantization_config,
 )
+from sglang.multimodal_gen.runtime.layers.quantization.fp8 import Fp8Config
+from sglang.multimodal_gen.runtime.server_args import ServerArgs
 from sglang.multimodal_gen.runtime.utils.logging_utils import init_logger
 
 logger = init_logger(__name__)
@@ -109,6 +111,16 @@ def find_quant_modelslim_config(model_config, component_model_path):
     return quant_cfg
 
 
+def resolve_runtime_quant_config(quantization_method: str) -> QuantizationConfig:
+    if quantization_method == "fp8":
+        logger.info("Runtime FP8 quantization enabled.")
+        return Fp8Config(
+            is_checkpoint_fp8_serialized=False,
+            activation_scheme="dynamic",
+        )
+    return None
+
+
 def replace_prefix(key: str, prefix_mapping: dict[str, str]) -> str:
     for prefix, new_prefix in prefix_mapping.items():
         if key.startswith(prefix):
@@ -121,7 +133,12 @@ def get_quant_config(
     component_model_path: str,
     packed_modules_mapping: Dict[str, List[str]] = {},
     remap_prefix: Dict[str, str] | None = None,
+    server_args: ServerArgs | None = None,
 ) -> QuantizationConfig:
+
+    if server_args and server_args.quantization:
+        return resolve_runtime_quant_config(server_args.quantization)
+
     quant_cfg = find_quant_modelslim_config(model_config, component_model_path)
     if quant_cfg is not None:
         quant_cls = _load_quant_cls(quant_cfg)