From e464bd98d6b912dc9f8809bf71fa9cdfde3f0cbe Mon Sep 17 00:00:00 2001 From: alibhutto69 Date: Sun, 27 Sep 2026 13:48:26 +0500 Subject: [PATCH 1/2] fix(metrics): retain components for resolved quantization configs Signed-off-by: alibhutto69 --- tests/v1/metrics/test_perf_metrics.py | 180 +++++++++++++++++++++++--- vllm/v1/metrics/perf.py | 113 +++++++++++----- 2 files changed, 247 insertions(+), 46 deletions(-) diff --git a/tests/v1/metrics/test_perf_metrics.py b/tests/v1/metrics/test_perf_metrics.py index c0997ec459bf..a109037e4162 100644 --- a/tests/v1/metrics/test_perf_metrics.py +++ b/tests/v1/metrics/test_perf_metrics.py @@ -2,6 +2,7 @@ # SPDX-FileCopyrightText: Copyright contributors to the vLLM project """Tests for the analytic estimators in metrics/flops.py.""" +import logging import types from types import SimpleNamespace @@ -16,6 +17,7 @@ from transformers.models.qwen3_moe.configuration_qwen3_moe import Qwen3MoeConfig from vllm.config.model import ModelConfig, get_hf_text_config +from vllm.config.quantization import resolve_quantization_config from vllm.transformers_utils.model_arch_config_convertor import ( MODEL_ARCH_CONFIG_CONVERTORS, ModelArchConfigConvertorBase, @@ -26,7 +28,6 @@ BaseConfigParser, ExecutionContext, FfnMetrics, - InvalidComponent, MLAAttentionMetrics, ModelMetrics, ParsedArgs, @@ -1024,26 +1025,173 @@ def get_name(self): ) -def test_quantization_config_parser_unknown_method(): - """Test that an unrecognized quant method raises InvalidComponent.""" +@pytest.mark.parametrize( + "quant_method,weight_bits,expected_bytes", + [ + ("auto_gptq", 4, 0.5), + ("auto_gptq", 8, 1), + ("auto_awq", 4, 0.5), + ], +) +def test_quantized_model_metrics_keep_transformer_components( + quant_method, weight_bits, expected_bytes +): + """Resolved quantization names must not silently remove attention or FFN.""" + quant_config = SimpleNamespace( + get_name=lambda: quant_method, weight_bits=weight_bits + ) + config = create_mock_vllm_config(Qwen3Config(), quant_config=quant_config) + metrics = ModelMetrics(config) + components = {m.component_type(): m for m in metrics.metrics} + assert set(components) == {"attn", "ffn", "unembed"} + for name in ("attn", "ffn"): + assert components[name].weight_byte_size == expected_bytes - class MockQuantConfig: - def get_name(self): - return "unknown_quant_method" + ctx = ExecutionContext.from_single_request(1, 1024, is_prefill=False) + unquantized = ModelMetrics(create_mock_vllm_config(Qwen3Config())) + assert metrics.get_num_flops(ctx) == unquantized.get_num_flops(ctx) + assert 0 < metrics.get_read_bytes(ctx) < unquantized.get_read_bytes(ctx) - hf_config = Qwen3Config( - hidden_size=2048, - num_attention_heads=16, - intermediate_size=8192, - num_hidden_layers=1, + +@pytest.mark.parametrize( + "quant_method,expert_dtype,attention_bytes,ffn_bytes", + [ + ("gpt_oss_mxfp4", None, 2, 0.5), + ("deepseek_v4_fp8", None, 1, 0.5), + ("deepseek_v4_fp8", "fp4", 1, 0.5), + ("deepseek_v4_fp8", "fp8", 1, 1), + ], +) +def test_checkpoint_quantization_respects_linear_and_expert_precision( + quant_method, expert_dtype, attention_bytes, ffn_bytes +): + """The checkpoint method's name does not determine every layer's precision.""" + hf_config = Qwen3MoeConfig() + if expert_dtype is not None: + hf_config.expert_dtype = expert_dtype + config = create_mock_vllm_config( + hf_config, quant_config=SimpleNamespace(get_name=lambda: quant_method) ) - vllm_config = create_mock_vllm_config(hf_config, quant_config=MockQuantConfig()) + metrics = ModelMetrics(config) + components = {m.component_type(): m for m in metrics.metrics} + assert set(components) == {"attn", "ffn", "unembed"} + assert components["attn"].weight_byte_size == attention_bytes + assert components["ffn"].weight_byte_size == ffn_bytes - with pytest.raises(InvalidComponent): - AttentionMetrics.get_parser().parse(vllm_config) - with pytest.raises(InvalidComponent): - FfnMetrics.get_parser().parse(vllm_config) +@pytest.mark.parametrize("quant_method", ["gpt_oss_mxfp4", "deepseek_v4_fp8"]) +def test_mixed_checkpoint_ffn_precision_falls_back(quant_method): + hf_config = Qwen3MoeConfig() + hf_config.n_shared_experts = 1 + config = create_mock_vllm_config( + hf_config, quant_config=SimpleNamespace(get_name=lambda: quant_method) + ) + assert FfnMetrics.from_vllm_config(config).weight_byte_size == 2 + + +@pytest.mark.parametrize( + "quant_method", ["unknown_quant_method", "humming", "modelopt_mixed"] +) +@pytest.mark.parametrize( + "model_dtype,expected_bytes", [("bfloat16", 2), ("float32", 4)] +) +def test_quantization_config_parser_unknown_method( + quant_method, model_dtype, expected_bytes, caplog, monkeypatch +): + """Unsupported sizing keeps all components and warns about the estimate.""" + monkeypatch.setattr(logging.getLogger("vllm"), "propagate", True) + quant_config = SimpleNamespace(get_name=lambda: quant_method) + config = create_mock_vllm_config( + Qwen3Config(), model_dtype=model_dtype, quant_config=quant_config + ) + metrics = ModelMetrics(config) + components = {m.component_type(): m for m in metrics.metrics} + assert set(components) == {"attn", "ffn", "unembed"} + assert all(m.weight_byte_size == expected_bytes for m in components.values()) + assert quant_method in caplog.text + assert "model dtype" in caplog.text + warnings = [r for r in caplog.records if r.levelno == logging.WARNING] + assert len(warnings) == 1 + + ctx = ExecutionContext.from_single_request(1, 1024, is_prefill=False) + unquantized = ModelMetrics( + create_mock_vllm_config(Qwen3Config(), model_dtype=model_dtype) + ) + assert metrics.get_num_flops(ctx) == unquantized.get_num_flops(ctx) + assert metrics.get_read_bytes(ctx) == unquantized.get_read_bytes(ctx) + + +@pytest.mark.parametrize( + "shorthand,expected_bytes", + [ + ("fp8_per_tensor", 1), + ("fp8_per_block", 1), + ("fp8_per_channel", 1), + ("mxfp8", 1), + ("mxfp4", 0.5), + ], +) +def test_online_quantization_metrics_use_resolved_weight_precision( + shorthand, expected_bytes +): + """Online FP4 storage packs two weights per byte; it is not FP8.""" + quant_config = SimpleNamespace( + get_name=lambda: "online", + args=resolve_quantization_config(shorthand, {}), + ) + config = create_mock_vllm_config(Qwen3Config(), quant_config=quant_config) + for component in (AttentionMetrics, FfnMetrics): + assert component.from_vllm_config(config).weight_byte_size == expected_bytes + + +@pytest.mark.parametrize( + "shorthand,expected_bytes", + [("int8_per_channel_weight_only", 1), ("nvfp4_per_token", 0.5)], +) +def test_online_moe_only_quantization_preserves_linear_precision( + shorthand, expected_bytes +): + quant_config = SimpleNamespace( + get_name=lambda: "online", + args=resolve_quantization_config(shorthand, None), + ) + config = create_mock_vllm_config(Qwen3MoeConfig(), quant_config=quant_config) + assert AttentionMetrics.from_vllm_config(config).weight_byte_size == 2 + assert FfnMetrics.from_vllm_config(config).weight_byte_size == expected_bytes + + +@pytest.mark.parametrize( + "overrides", + [ + {"targets": {"*.q_proj": "fp8_per_tensor"}}, + {"linear": "fp8_per_tensor", "ignore": ["*.q_proj"]}, + ], +) +def test_online_layer_overrides_keep_components_with_dtype_fallback(overrides): + quant_config = SimpleNamespace( + get_name=lambda: "online", + args=resolve_quantization_config("online", overrides), + ) + config = create_mock_vllm_config(Qwen3Config(), quant_config=quant_config) + metrics = ModelMetrics(config) + assert {m.component_type() for m in metrics.metrics} == {"attn", "ffn", "unembed"} + assert all(m.weight_byte_size == 2 for m in metrics.metrics) + + +def test_online_mixed_dense_and_moe_precision_falls_back(caplog, monkeypatch): + """A single FFN weight size cannot describe differently quantized layers.""" + monkeypatch.setattr(logging.getLogger("vllm"), "propagate", True) + quant_config = SimpleNamespace( + get_name=lambda: "online", + args=resolve_quantization_config("int8_per_channel_weight_only", None), + ) + hf_config = Qwen3MoeConfig() + hf_config.interleave_moe_layer_step = 2 + config = create_mock_vllm_config(hf_config, quant_config=quant_config) + metrics = FfnMetrics.from_vllm_config(config) + assert metrics.num_moe_layers == hf_config.num_hidden_layers // 2 + assert metrics.weight_byte_size == 2 + assert "model dtype" in caplog.text def test_quantized_model_metrics_aggregation(): diff --git a/vllm/v1/metrics/perf.py b/vllm/v1/metrics/perf.py index a2b3f534d88c..4b4cf4fb202a 100644 --- a/vllm/v1/metrics/perf.py +++ b/vllm/v1/metrics/perf.py @@ -10,7 +10,7 @@ from abc import ABC, abstractmethod from collections.abc import Iterable from dataclasses import asdict, dataclass, field -from typing import Any, Protocol, Self +from typing import TYPE_CHECKING, Any, Literal, Protocol, Self, cast import prometheus_client import torch @@ -19,6 +19,8 @@ import vllm.envs as envs from vllm.config import VllmConfig from vllm.logger import init_logger +from vllm.model_executor.layers.quantization.utils.quant_utils import FP4_DTYPE +from vllm.scalar_type import ScalarType from vllm.utils.torch_utils import ( STR_DTYPE_TO_TORCH_DTYPE, get_dtype_size, @@ -27,6 +29,13 @@ from vllm.v1.core.sched.output import SchedulerOutput from vllm.v1.metrics.utils import create_metric_per_engine +if TYPE_CHECKING: + from vllm.model_executor.layers.quantization.auto_awq import AutoAWQConfig + from vllm.model_executor.layers.quantization.auto_gptq import AutoGPTQConfig + from vllm.model_executor.layers.quantization.online.base import ( + OnlineQuantizationConfig, + ) + logger = init_logger(__name__) @@ -43,9 +52,8 @@ class InvalidComponent(Exception): # FfnQuantizationConfigParser to determine the weight_byte_size for # flops/memory estimation. # -# NOTE: Methods like GPTQ support variable bit-widths -# (e.g., 4-bit and 8-bit). We default to 4-bit (0.5 bytes) since this -# is by far the most common configuration. +# Legacy GPTQ aliases default to 4-bit (0.5 bytes). Resolved AutoGPTQ +# and AutoAWQ configs use their configured bit widths below. _QUANT_WEIGHT_BYTE_SIZE: dict[str, float] = { # FP8 methods (1 byte per weight) "fp8": 1, @@ -71,6 +79,66 @@ class InvalidComponent(Exception): } +def _get_quantized_weight_byte_size( + vllm_config: VllmConfig, + default_byte_size: float, + layer_kinds: tuple[Literal["linear", "moe"], ...] = ("linear",), +) -> float: + cfg = vllm_config.quant_config + if cfg is None: + return default_byte_size + + quant_method = cfg.get_name() + if quant_method in ("auto_gptq", "auto_awq"): + return cast("AutoGPTQConfig | AutoAWQConfig", cfg).weight_bits / 8 + if quant_method in _QUANT_WEIGHT_BYTE_SIZE: + return _QUANT_WEIGHT_BYTE_SIZE[quant_method] + byte_sizes: set[float] = set() + if quant_method == "gpt_oss_mxfp4": + byte_sizes = { + 0.5 if kind == "moe" else default_byte_size for kind in layer_kinds + } + elif quant_method == "deepseek_v4_fp8": + expert_dtype = getattr( + vllm_config.model_config.hf_text_config, "expert_dtype", "fp4" + ) + if expert_dtype in ("fp4", "fp8"): + byte_sizes = { + 0.5 if kind == "moe" and expert_dtype == "fp4" else 1 + for kind in layer_kinds + } + elif quant_method == "online": + quant_args = cast("OnlineQuantizationConfig", cfg).args + # A component has one weight size, so per-layer overrides cannot be + # represented by the current estimator. + if not quant_args.targets and not quant_args.ignore: + for kind in layer_kinds: + spec = getattr(quant_args, kind) + if spec is None or spec.weight is None: + byte_sizes.add(default_byte_size) + else: + dtype = spec.weight.dtype + if isinstance(dtype, ScalarType): + byte_sizes.add(dtype.size_bits / 8) + elif dtype == FP4_DTYPE: + # FP4 storage packs two logical weights into one uint8. + byte_sizes.add(0.5) + else: + byte_sizes.add(get_dtype_size(dtype)) + if len(byte_sizes) == 1: + return byte_sizes.pop() + + logger.warning_once( + "Cannot determine a uniform weight size for quantization method %s " + "(%s). Using model dtype (%s bytes) for the MFU/MBU estimate; " + "memory traffic may be overestimated.", + quant_method, + "/".join(layer_kinds), + default_byte_size, + ) + return default_byte_size + + #### Basic Data Types #### @@ -459,19 +527,9 @@ class AttentionQuantizationConfigParser(Parser): def parse(self, args: ParsedArgs, vllm_config: VllmConfig) -> ParsedArgs: """Override weight_byte_size based on the active quantization method.""" - cfg = vllm_config.quant_config - - if cfg is None: - return args - - quant_method = cfg.get_name() - if quant_method in _QUANT_WEIGHT_BYTE_SIZE: - args.weight_byte_size = _QUANT_WEIGHT_BYTE_SIZE[quant_method] - else: - raise InvalidComponent( - f"Unsupported quantization method for attention metrics: {quant_method}" - ) - + args.weight_byte_size = _get_quantized_weight_byte_size( + vllm_config, args.weight_byte_size + ) return args @@ -1116,19 +1174,14 @@ class FfnQuantizationConfigParser(Parser): def parse(self, args: ParsedArgs, vllm_config: VllmConfig) -> ParsedArgs: """Override weight_byte_size based on the active FFN quantization method.""" - cfg = vllm_config.quant_config - - if cfg is None: - return args - - quant_method = cfg.get_name() - if quant_method in _QUANT_WEIGHT_BYTE_SIZE: - args.weight_byte_size = _QUANT_WEIGHT_BYTE_SIZE[quant_method] - else: - raise InvalidComponent( - f"Unsupported quantization method for FFN metrics: {quant_method}" - ) - + layer_kinds: tuple[Literal["linear", "moe"], ...] = ("linear",) + if args.num_moe_layers > 0: + layer_kinds = ("moe",) + if args.num_moe_layers < args.num_hidden_layers or args.num_shared_experts: + layer_kinds = ("linear", "moe") + args.weight_byte_size = _get_quantized_weight_byte_size( + vllm_config, args.weight_byte_size, layer_kinds + ) return args From ba29f3aa76508fdc9682d947f3089721965d455f Mon Sep 17 00:00:00 2001 From: alibhutto69 Date: Sun, 27 Sep 2026 20:13:46 +0500 Subject: [PATCH 2/2] fix(metrics): warn when checkpoint layer overrides prevent uniform sizing Signed-off-by: alibhutto69 --- tests/v1/metrics/test_perf_metrics.py | 44 ++++++++++++++++++++++++++- vllm/v1/metrics/perf.py | 16 +++++++--- 2 files changed, 54 insertions(+), 6 deletions(-) diff --git a/tests/v1/metrics/test_perf_metrics.py b/tests/v1/metrics/test_perf_metrics.py index a109037e4162..a88aba95ba14 100644 --- a/tests/v1/metrics/test_perf_metrics.py +++ b/tests/v1/metrics/test_perf_metrics.py @@ -1038,7 +1038,11 @@ def test_quantized_model_metrics_keep_transformer_components( ): """Resolved quantization names must not silently remove attention or FFN.""" quant_config = SimpleNamespace( - get_name=lambda: quant_method, weight_bits=weight_bits + get_name=lambda: quant_method, + weight_bits=weight_bits, + dynamic={}, + modules_in_block_to_quantize=[], + modules_to_not_convert=[], ) config = create_mock_vllm_config(Qwen3Config(), quant_config=quant_config) metrics = ModelMetrics(config) @@ -1053,6 +1057,44 @@ def test_quantized_model_metrics_keep_transformer_components( assert 0 < metrics.get_read_bytes(ctx) < unquantized.get_read_bytes(ctx) +@pytest.mark.parametrize( + "quant_method,overrides", + [ + ("gptq", {"dynamic": {".*q_proj": {"bits": 8}}}), + ("gptq", {"dynamic": {"-:.*q_proj": {}}}), + ("gptq", {"modules_in_block_to_quantize": ["q_proj"]}), + ("awq", {"modules_to_not_convert": ["q_proj"]}), + ], +) +def test_checkpoint_layer_overrides_use_warned_dtype_fallback( + quant_method, overrides, caplog, monkeypatch +): + from vllm.model_executor.layers.quantization.auto_awq import AutoAWQConfig + from vllm.model_executor.layers.quantization.auto_gptq import AutoGPTQConfig + from vllm.v1.metrics.perf import logger + + monkeypatch.setattr(logging.getLogger("vllm"), "propagate", True) + # Deduplication is tested separately; each override must reach the warning. + monkeypatch.setattr(logger, "warning_once", logger.warning) + config_cls = AutoGPTQConfig if quant_method == "gptq" else AutoAWQConfig + quant_config = config_cls.from_config( + { + "bits": 4, + "group_size": 128, + "desc_act": False, + "sym": True, + "zero_point": True, + **overrides, + } + ) + config = create_mock_vllm_config(Qwen3Config(), quant_config=quant_config) + metrics = ModelMetrics(config) + components = {m.component_type(): m for m in metrics.metrics} + assert set(components) == {"attn", "ffn", "unembed"} + assert all(m.weight_byte_size == 2 for m in components.values()) + assert "model dtype" in caplog.text + + @pytest.mark.parametrize( "quant_method,expert_dtype,attention_bytes,ffn_bytes", [ diff --git a/vllm/v1/metrics/perf.py b/vllm/v1/metrics/perf.py index 4b4cf4fb202a..76f8ad315a1f 100644 --- a/vllm/v1/metrics/perf.py +++ b/vllm/v1/metrics/perf.py @@ -89,12 +89,18 @@ def _get_quantized_weight_byte_size( return default_byte_size quant_method = cfg.get_name() - if quant_method in ("auto_gptq", "auto_awq"): - return cast("AutoGPTQConfig | AutoAWQConfig", cfg).weight_bits / 8 - if quant_method in _QUANT_WEIGHT_BYTE_SIZE: - return _QUANT_WEIGHT_BYTE_SIZE[quant_method] byte_sizes: set[float] = set() - if quant_method == "gpt_oss_mxfp4": + if quant_method == "auto_gptq": + gptq_cfg = cast("AutoGPTQConfig", cfg) + if not gptq_cfg.dynamic and not gptq_cfg.modules_in_block_to_quantize: + return gptq_cfg.weight_bits / 8 + elif quant_method == "auto_awq": + awq_cfg = cast("AutoAWQConfig", cfg) + if not awq_cfg.modules_to_not_convert: + return awq_cfg.weight_bits / 8 + elif quant_method in _QUANT_WEIGHT_BYTE_SIZE: + return _QUANT_WEIGHT_BYTE_SIZE[quant_method] + elif quant_method == "gpt_oss_mxfp4": byte_sizes = { 0.5 if kind == "moe" else default_byte_size for kind in layer_kinds }