From 6da11eaeeb4a4cc257d8eee5b305e69031944d78 Mon Sep 17 00:00:00 2001 From: Brayden Zhong Date: Thu, 4 Jun 2026 08:54:18 -0700 Subject: [PATCH 1/2] more --- python/sglang/srt/model_executor/model_runner.py | 13 ------------- 1 file changed, 13 deletions(-) diff --git a/python/sglang/srt/model_executor/model_runner.py b/python/sglang/srt/model_executor/model_runner.py index ba7ec25d23d3..3adbb91a5111 100644 --- a/python/sglang/srt/model_executor/model_runner.py +++ b/python/sglang/srt/model_executor/model_runner.py @@ -1418,19 +1418,6 @@ def load_model(self): f"mem usage={self.weight_load_mem_usage:.2f} GB." ) - # TODO: Make sure all models have `quant_config` attribute, and all online quantization methods register which layers they actually quantize. - if ( - hasattr(self.model, "quant_config") - and hasattr(self.model.quant_config, "quantized_layers") - and self.server_args.quantization is not None - ): - layer_types, quantized_layers_count = ( - self.model.quant_config.quantized_layers - ) - logger.info( - f"Online {self.server_args.quantization} quantization: quantized {quantized_layers_count} layers of types: {layer_types}" - ) - if self.server_args.debug_tensor_dump_output_folder is not None: dump_folder = self.server_args.debug_tensor_dump_output_folder if self.spec_algorithm.is_eagle(): From 3b194cc10ebf16dcbc7654713bf5de31748c9d5c Mon Sep 17 00:00:00 2001 From: Brayden Zhong Date: Thu, 4 Jun 2026 14:20:25 -0700 Subject: [PATCH 2/2] more --- python/sglang/srt/model_executor/model_runner.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/python/sglang/srt/model_executor/model_runner.py b/python/sglang/srt/model_executor/model_runner.py index 3adbb91a5111..dc101837d4a8 100644 --- a/python/sglang/srt/model_executor/model_runner.py +++ b/python/sglang/srt/model_executor/model_runner.py @@ -1418,6 +1418,21 @@ def load_model(self): f"mem usage={self.weight_load_mem_usage:.2f} GB." ) + # TODO: Make sure all models have `quant_config` attribute, and all online quantization methods register which layers they actually quantize. + # TODO: Move this online-quantization reporting out of ModelRunner. + quantized_layers = getattr( + getattr(self.model, "quant_config", None), "quantized_layers", None + ) + if ( + self.server_args.quantization is not None + and isinstance(quantized_layers, tuple) + and len(quantized_layers) == 2 + ): + layer_types, quantized_layers_count = quantized_layers + logger.info( + f"Online {self.server_args.quantization} quantization: quantized {quantized_layers_count} layers of types: {layer_types}" + ) + if self.server_args.debug_tensor_dump_output_folder is not None: dump_folder = self.server_args.debug_tensor_dump_output_folder if self.spec_algorithm.is_eagle():