From d2f7be871da877a57dd673e03bd80744040e8337 Mon Sep 17 00:00:00 2001 From: Xingyu Liu Date: Thu, 22 Jan 2026 17:47:47 -0800 Subject: [PATCH 1/8] refactor rope related Signed-off-by: Xingyu Liu --- vllm/config/model.py | 23 +++++++------ vllm/config/model_arch.py | 15 ++++++++ .../model_arch_config_convertor.py | 34 ++++++++++++++++++- 3 files changed, 60 insertions(+), 12 deletions(-) diff --git a/vllm/config/model.py b/vllm/config/model.py index 883e0b17ef0e..dbb5ebbb21f2 100644 --- a/vllm/config/model.py +++ b/vllm/config/model.py @@ -33,8 +33,6 @@ try_get_dense_modules, try_get_generation_config, try_get_tokenizer_config, - uses_mrope, - uses_xdrope_dim, ) from vllm.transformers_utils.gguf_utils import ( is_gguf, @@ -1407,11 +1405,11 @@ def uses_alibi(self) -> bool: @property def uses_mrope(self) -> bool: - return uses_mrope(self.hf_config) + return self.model_arch_config.uses_mrope @property def uses_xdrope_dim(self) -> int: - return uses_xdrope_dim(self.hf_config) + return self.model_arch_config.uses_xdrope_dim @property def is_multimodal_model(self) -> bool: @@ -1962,15 +1960,16 @@ def _get_and_verify_max_len( ) derived_max_model_len = default_max_len + # Get rope_parameters from model_arch_config # In Transformers v5 rope_parameters could be TypedDict or dict[str, TypedDict]. # To simplify the verification, we convert it to dict[str, TypedDict]. - rope_parameters = getattr(hf_config, "rope_parameters", None) + rope_parameters = model_arch_config.rope_parameters if rope_parameters and not is_rope_parameters_nested(rope_parameters): rope_parameters = {"": rope_parameters} # NOTE(woosuk): Gemma3's max_model_len (128K) is already scaled by RoPE # scaling, so we skip applying the scaling factor again. - if rope_parameters is not None and "gemma3" not in hf_config.model_type: + if rope_parameters is not None and "gemma3" not in model_arch_config.model_type: scaling_factor = 1.0 for rp in rope_parameters.values(): # No need to consider "type" key because of patch_rope_parameters when @@ -1999,11 +1998,11 @@ def _get_and_verify_max_len( if rope_parameters is not None and any( rp["rope_type"] == "longrope" for rp in rope_parameters.values() ): - max_model_len = int( - getattr( - hf_config, "original_max_position_embeddings", derived_max_model_len - ) - ) + original_max_pos = model_arch_config.original_max_position_embeddings + if original_max_pos is not None: + max_model_len = int(original_max_pos) + else: + max_model_len = int(derived_max_model_len) else: max_model_len = int(derived_max_model_len) max_model_len = current_platform.check_max_model_len(max_model_len) @@ -2014,6 +2013,8 @@ def _get_and_verify_max_len( # Some models might have a separate key for specifying model_max_length # that will be bigger than derived_max_model_len. We compare user input # with model_max_length and allow this override when it's smaller. + # NOTE: model_max_length is not consolidated into model_arch_config + # as it's primarily used for tokenizer limits, not model architecture. model_max_length = getattr(hf_config, "model_max_length", None) if model_max_length is None or max_model_len > model_max_length: msg = ( diff --git a/vllm/config/model_arch.py b/vllm/config/model_arch.py index d55e2a3399b3..1a78b50b7068 100644 --- a/vllm/config/model_arch.py +++ b/vllm/config/model_arch.py @@ -55,3 +55,18 @@ class ModelArchitectureConfig: derived_max_model_len_and_key: tuple[float, str | None] """Derived maximum model length and key from the hf config.""" + + # RoPE-related fields + uses_mrope: bool + """Whether the model uses M-RoPE (multi-dimensional rotary position embedding).""" + + uses_xdrope_dim: int + """Number of dimensions for XD-RoPE. 0 if not used.""" + + rope_parameters: dict[str, Any] | None + """RoPE parameters dictionary containing RoPE configuration. + Can be None if the model doesn't use RoPE.""" + + original_max_position_embeddings: int | None + """Original maximum position embeddings before any RoPE scaling. + Used for models with extended context via RoPE scaling.""" diff --git a/vllm/transformers_utils/model_arch_config_convertor.py b/vllm/transformers_utils/model_arch_config_convertor.py index 6df4bb64dceb..b680cbc533ea 100644 --- a/vllm/transformers_utils/model_arch_config_convertor.py +++ b/vllm/transformers_utils/model_arch_config_convertor.py @@ -1,7 +1,7 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project -from typing import final +from typing import Any, final import torch from safetensors.torch import _TYPES as _SAFETENSORS_TO_TORCH_DTYPE @@ -15,6 +15,8 @@ from vllm.logger import init_logger from vllm.transformers_utils.config import ( try_get_safetensors_metadata, + uses_mrope, + uses_xdrope_dim, ) from vllm.utils.torch_utils import common_broadcastable_dtype @@ -241,6 +243,31 @@ def derive_max_model_len_and_key(self) -> tuple[float, str | None]: derived_max_model_len = tmp_max_len return derived_max_model_len, max_len_key + def get_uses_mrope(self) -> bool: + return uses_mrope(self.hf_config) + + def get_uses_xdrope_dim(self) -> int: + """Get the number of dimensions for XD-RoPE. Returns 0 if not used.""" + return uses_xdrope_dim(self.hf_config) + + def get_rope_parameters(self) -> dict[str, Any] | None: + """Get the RoPE parameters from the config.""" + rope_params = getattr(self.hf_text_config, "rope_parameters", None) + if rope_params is not None: + return dict(rope_params) + return None + + def get_original_max_position_embeddings(self) -> int | None: + """Get the original max position embeddings before RoPE scaling.""" + ompe = getattr(self.hf_text_config, "original_max_position_embeddings", None) + if ompe is not None: + return ompe + # Also check rope_parameters for this field + rope_params = getattr(self.hf_text_config, "rope_parameters", None) + if rope_params is not None: + return rope_params.get("original_max_position_embeddings") + return None + def convert(self) -> ModelArchitectureConfig: model_arch_config = ModelArchitectureConfig( architectures=self.get_architectures(), @@ -256,6 +283,11 @@ def convert(self) -> ModelArchitectureConfig: quantization_config=self.get_quantization_config(), is_deepseek_mla=self.is_deepseek_mla(), derived_max_model_len_and_key=self.derive_max_model_len_and_key(), + # RoPE-related fields + uses_mrope=self.get_uses_mrope(), + uses_xdrope_dim=self.get_uses_xdrope_dim(), + rope_parameters=self.get_rope_parameters(), + original_max_position_embeddings=self.get_original_max_position_embeddings(), ) return model_arch_config From 92a907625856ad8a9239e9ae2af5bd1da89b704e Mon Sep 17 00:00:00 2001 From: Xingyu Liu Date: Fri, 23 Jan 2026 15:24:35 -0800 Subject: [PATCH 2/8] refactor rope_parameters Signed-off-by: Xingyu Liu --- vllm/config/model.py | 40 ++------ vllm/config/model_arch.py | 30 +++--- .../model_arch_config_convertor.py | 98 ++++++++++++++----- 3 files changed, 97 insertions(+), 71 deletions(-) diff --git a/vllm/config/model.py b/vllm/config/model.py index dbb5ebbb21f2..ab22d693e510 100644 --- a/vllm/config/model.py +++ b/vllm/config/model.py @@ -29,7 +29,6 @@ get_pooling_config, get_sentence_transformer_tokenizer_config, is_encoder_decoder, - is_rope_parameters_nested, try_get_dense_modules, try_get_generation_config, try_get_tokenizer_config, @@ -1918,9 +1917,10 @@ def _get_and_verify_max_len( encoder_config: Any | None = None, ) -> int: """Get and verify the model's maximum length.""" - (derived_max_model_len, max_len_key) = ( - model_arch_config.derived_max_model_len_and_key - ) + # Get pre-computed derived max model len info (includes RoPE scaling) + max_len_info = model_arch_config.derived_max_model_len_info + derived_max_model_len = max_len_info.derived_max_model_len + max_len_key = max_len_info.max_len_key # If sliding window is manually disabled, max_length should be less # than the sliding window length in the model config. @@ -1960,32 +1960,6 @@ def _get_and_verify_max_len( ) derived_max_model_len = default_max_len - # Get rope_parameters from model_arch_config - # In Transformers v5 rope_parameters could be TypedDict or dict[str, TypedDict]. - # To simplify the verification, we convert it to dict[str, TypedDict]. - rope_parameters = model_arch_config.rope_parameters - if rope_parameters and not is_rope_parameters_nested(rope_parameters): - rope_parameters = {"": rope_parameters} - - # NOTE(woosuk): Gemma3's max_model_len (128K) is already scaled by RoPE - # scaling, so we skip applying the scaling factor again. - if rope_parameters is not None and "gemma3" not in model_arch_config.model_type: - scaling_factor = 1.0 - for rp in rope_parameters.values(): - # No need to consider "type" key because of patch_rope_parameters when - # loading HF config - rope_type = rp["rope_type"] - - if rope_type not in ("su", "longrope", "llama3"): - # NOTE: rope_type == "default" does not define factor https://github.com/huggingface/transformers/blob/v4.45.2/src/transformers/modeling_rope_utils.py - # NOTE: This assumes all layer types have the same scaling factor. - scaling_factor = rp.get("factor", scaling_factor) - - if rope_type == "yarn": - derived_max_model_len = rp["original_max_position_embeddings"] - # Do this outside loop since all layer types should have the same scaling - derived_max_model_len *= scaling_factor - if encoder_config and "max_seq_length" in encoder_config: derived_max_model_len = encoder_config["max_seq_length"] @@ -1995,10 +1969,8 @@ def _get_and_verify_max_len( if max_model_len is None or max_model_len == -1: # For LongRoPE, default to original_max_position_embeddings to avoid # performance degradation for shorter sequences - if rope_parameters is not None and any( - rp["rope_type"] == "longrope" for rp in rope_parameters.values() - ): - original_max_pos = model_arch_config.original_max_position_embeddings + if max_len_info.is_longrope: + original_max_pos = max_len_info.original_max_position_embeddings if original_max_pos is not None: max_model_len = int(original_max_pos) else: diff --git a/vllm/config/model_arch.py b/vllm/config/model_arch.py index 1a78b50b7068..b286ac445fd8 100644 --- a/vllm/config/model_arch.py +++ b/vllm/config/model_arch.py @@ -1,6 +1,6 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project -from typing import Any +from typing import Any, NamedTuple from pydantic import ConfigDict from pydantic.dataclasses import dataclass @@ -10,6 +10,22 @@ logger = init_logger(__name__) +class DerivedMaxModelLenInfo(NamedTuple): + """Information about the derived maximum model length.""" + + derived_max_model_len: float + """The derived maximum model length after applying RoPE scaling.""" + + max_len_key: str | None + """The key in the config that was used to derive the max length.""" + + is_longrope: bool + """Whether the model uses LongRoPE (affects default max_model_len selection).""" + + original_max_position_embeddings: int | None + """Original max position embeddings before RoPE scaling (for LongRoPE models).""" + + @dataclass(config=ConfigDict(arbitrary_types_allowed=True)) class ModelArchitectureConfig: """ @@ -53,8 +69,8 @@ class ModelArchitectureConfig: is_deepseek_mla: bool """Whether the model is a DeepSeek MLA model.""" - derived_max_model_len_and_key: tuple[float, str | None] - """Derived maximum model length and key from the hf config.""" + derived_max_model_len_info: DerivedMaxModelLenInfo + """Derived maximum model length information including RoPE scaling.""" # RoPE-related fields uses_mrope: bool @@ -62,11 +78,3 @@ class ModelArchitectureConfig: uses_xdrope_dim: int """Number of dimensions for XD-RoPE. 0 if not used.""" - - rope_parameters: dict[str, Any] | None - """RoPE parameters dictionary containing RoPE configuration. - Can be None if the model doesn't use RoPE.""" - - original_max_position_embeddings: int | None - """Original maximum position embeddings before any RoPE scaling. - Used for models with extended context via RoPE scaling.""" diff --git a/vllm/transformers_utils/model_arch_config_convertor.py b/vllm/transformers_utils/model_arch_config_convertor.py index b680cbc533ea..f25f59f1da46 100644 --- a/vllm/transformers_utils/model_arch_config_convertor.py +++ b/vllm/transformers_utils/model_arch_config_convertor.py @@ -9,11 +9,13 @@ from vllm import envs from vllm.config.model_arch import ( + DerivedMaxModelLenInfo, ModelArchitectureConfig, ) from vllm.config.utils import getattr_iter from vllm.logger import init_logger from vllm.transformers_utils.config import ( + is_rope_parameters_nested, try_get_safetensors_metadata, uses_mrope, uses_xdrope_dim, @@ -208,8 +210,15 @@ def is_deepseek_mla(self) -> bool: ) return False - def derive_max_model_len_and_key(self) -> tuple[float, str | None]: - derived_max_model_len = float("inf") + def derive_max_model_len_info(self) -> DerivedMaxModelLenInfo: + """Derive maximum model length including RoPE scaling factors. + + This method computes the derived max model length by: + 1. Finding the base max length from various config keys + 2. Applying RoPE scaling factors (linear, dynamic, yarn, etc.) + 3. Detecting LongRoPE for special handling + """ + derived_max_model_len: float = float("inf") possible_keys = [ # OPT "max_position_embeddings", @@ -229,7 +238,7 @@ def derive_max_model_len_and_key(self) -> tuple[float, str | None]: "seq_len", ] # Choose the smallest "max_length" from the possible keys - max_len_key = None + max_len_key: str | None = None for key in possible_keys: max_len = getattr(self.hf_text_config, key, None) if max_len is not None: @@ -241,33 +250,72 @@ def derive_max_model_len_and_key(self) -> tuple[float, str | None]: if tmp_max_len := getattr(self.hf_text_config, "model_max_length", None): max_len_key = "model_max_length" derived_max_model_len = tmp_max_len - return derived_max_model_len, max_len_key + + # Get rope_parameters and apply RoPE scaling + rope_parameters = getattr(self.hf_text_config, "rope_parameters", None) + is_longrope = False + original_max_position_embeddings: int | None = None + + # Get original_max_position_embeddings from config or rope_parameters + original_max_position_embeddings = getattr( + self.hf_text_config, "original_max_position_embeddings", None + ) + if original_max_position_embeddings is None and rope_parameters is not None: + original_max_position_embeddings = rope_parameters.get( + "original_max_position_embeddings" + ) + + if rope_parameters is not None: + # In Transformers v5 rope_parameters could be TypedDict or + # dict[str, TypedDict]. Normalize to dict[str, TypedDict]. + if not is_rope_parameters_nested(rope_parameters): + rope_params_dict: dict[str, dict[str, Any]] = {"": rope_parameters} + else: + rope_params_dict = rope_parameters + + # Check if any layer uses longrope + is_longrope = any( + rp.get("rope_type") == "longrope" for rp in rope_params_dict.values() + ) + + # NOTE(woosuk): Gemma3's max_model_len (128K) is already scaled by RoPE + # scaling, so we skip applying the scaling factor again. + model_type = getattr(self.hf_config, "model_type", "") + if "gemma3" not in model_type: + scaling_factor = 1.0 + for rp in rope_params_dict.values(): + rope_type = rp.get("rope_type", "default") + + if rope_type not in ("su", "longrope", "llama3"): + # NOTE: rope_type == "default" does not define factor + # NOTE: This assumes all layer types have the same + # scaling factor. + scaling_factor = rp.get("factor", scaling_factor) + + if rope_type == "yarn": + # For yarn, use original_max_position_embeddings + # from rope_parameters + yarn_ompe = rp.get("original_max_position_embeddings") + if yarn_ompe is not None: + derived_max_model_len = yarn_ompe + + # Apply scaling factor outside the loop since all layer types + # should have the same scaling + derived_max_model_len *= scaling_factor + + return DerivedMaxModelLenInfo( + derived_max_model_len=derived_max_model_len, + max_len_key=max_len_key, + is_longrope=is_longrope, + original_max_position_embeddings=original_max_position_embeddings, + ) def get_uses_mrope(self) -> bool: return uses_mrope(self.hf_config) def get_uses_xdrope_dim(self) -> int: - """Get the number of dimensions for XD-RoPE. Returns 0 if not used.""" return uses_xdrope_dim(self.hf_config) - def get_rope_parameters(self) -> dict[str, Any] | None: - """Get the RoPE parameters from the config.""" - rope_params = getattr(self.hf_text_config, "rope_parameters", None) - if rope_params is not None: - return dict(rope_params) - return None - - def get_original_max_position_embeddings(self) -> int | None: - """Get the original max position embeddings before RoPE scaling.""" - ompe = getattr(self.hf_text_config, "original_max_position_embeddings", None) - if ompe is not None: - return ompe - # Also check rope_parameters for this field - rope_params = getattr(self.hf_text_config, "rope_parameters", None) - if rope_params is not None: - return rope_params.get("original_max_position_embeddings") - return None - def convert(self) -> ModelArchitectureConfig: model_arch_config = ModelArchitectureConfig( architectures=self.get_architectures(), @@ -282,12 +330,10 @@ def convert(self) -> ModelArchitectureConfig: num_experts=self.get_num_experts(), quantization_config=self.get_quantization_config(), is_deepseek_mla=self.is_deepseek_mla(), - derived_max_model_len_and_key=self.derive_max_model_len_and_key(), + derived_max_model_len_info=self.derive_max_model_len_info(), # RoPE-related fields uses_mrope=self.get_uses_mrope(), uses_xdrope_dim=self.get_uses_xdrope_dim(), - rope_parameters=self.get_rope_parameters(), - original_max_position_embeddings=self.get_original_max_position_embeddings(), ) return model_arch_config From e8ea1637ab5fb2cc3090fc08c298414d527f260f Mon Sep 17 00:00:00 2001 From: Xingyu Liu Date: Fri, 23 Jan 2026 16:19:57 -0800 Subject: [PATCH 3/8] handle longrope Signed-off-by: Xingyu Liu --- vllm/config/model.py | 17 ++++++------- vllm/config/model_arch.py | 24 ++++++++++--------- .../model_arch_config_convertor.py | 18 ++++++++++---- 3 files changed, 34 insertions(+), 25 deletions(-) diff --git a/vllm/config/model.py b/vllm/config/model.py index ab22d693e510..5fb4cf0aa19d 100644 --- a/vllm/config/model.py +++ b/vllm/config/model.py @@ -1919,8 +1919,8 @@ def _get_and_verify_max_len( """Get and verify the model's maximum length.""" # Get pre-computed derived max model len info (includes RoPE scaling) max_len_info = model_arch_config.derived_max_model_len_info - derived_max_model_len = max_len_info.derived_max_model_len - max_len_key = max_len_info.max_len_key + derived_max_model_len = max_len_info.derived + max_len_key = max_len_info.derived_key # If sliding window is manually disabled, max_length should be less # than the sliding window length in the model config. @@ -1967,15 +1967,12 @@ def _get_and_verify_max_len( # then use that derived from the model config as a default value. # When -1 is specified, the engine will later auto-fit to available memory. if max_model_len is None or max_model_len == -1: - # For LongRoPE, default to original_max_position_embeddings to avoid - # performance degradation for shorter sequences - if max_len_info.is_longrope: - original_max_pos = max_len_info.original_max_position_embeddings - if original_max_pos is not None: - max_model_len = int(original_max_pos) - else: - max_model_len = int(derived_max_model_len) + if max_len_info.default is not None: + # For LongRoPE, default to original_max_position_embeddings to avoid + # performance degradation for shorter sequences + max_model_len = int(max_len_info.default) else: + # Non-LongRoPE: use derived_max_model_len with caps applied max_model_len = int(derived_max_model_len) max_model_len = current_platform.check_max_model_len(max_model_len) diff --git a/vllm/config/model_arch.py b/vllm/config/model_arch.py index b286ac445fd8..1394f3efb999 100644 --- a/vllm/config/model_arch.py +++ b/vllm/config/model_arch.py @@ -13,17 +13,19 @@ class DerivedMaxModelLenInfo(NamedTuple): """Information about the derived maximum model length.""" - derived_max_model_len: float - """The derived maximum model length after applying RoPE scaling.""" - - max_len_key: str | None - """The key in the config that was used to derive the max length.""" - - is_longrope: bool - """Whether the model uses LongRoPE (affects default max_model_len selection).""" - - original_max_position_embeddings: int | None - """Original max position embeddings before RoPE scaling (for LongRoPE models).""" + derived: float + """Maximum supported sequence length after RoPE scaling. + Used for: + 1. Validation - user-specified max_model_len cannot exceed this. + 2. Default for non-LongRoPE models (with sliding_window/tokenizer caps).""" + + derived_key: str | None + """The config key used to derive the max length (for error messages).""" + + default: float | None + """For LongRoPE models only: original_max_position_embeddings. + Used as the default max_model_len to avoid performance degradation. + None for non-LongRoPE models (derived is used instead).""" @dataclass(config=ConfigDict(arbitrary_types_allowed=True)) diff --git a/vllm/transformers_utils/model_arch_config_convertor.py b/vllm/transformers_utils/model_arch_config_convertor.py index f25f59f1da46..a36fc241d377 100644 --- a/vllm/transformers_utils/model_arch_config_convertor.py +++ b/vllm/transformers_utils/model_arch_config_convertor.py @@ -303,11 +303,21 @@ def derive_max_model_len_info(self) -> DerivedMaxModelLenInfo: # should have the same scaling derived_max_model_len *= scaling_factor + # Compute default_max_model_len: + # For LongRoPE, use original_max_position_embeddings to avoid + # performance degradation for shorter sequences. + # For non-LongRoPE, return None (caller should use derived_max_model_len). + if is_longrope and original_max_position_embeddings is not None: + default_max_model_len: float | None = float( + original_max_position_embeddings + ) + else: + default_max_model_len = None + return DerivedMaxModelLenInfo( - derived_max_model_len=derived_max_model_len, - max_len_key=max_len_key, - is_longrope=is_longrope, - original_max_position_embeddings=original_max_position_embeddings, + derived=derived_max_model_len, + derived_key=max_len_key, + default=default_max_model_len, ) def get_uses_mrope(self) -> bool: From 77ff9878a7334ea15578f0fd01aa858b52acebc4 Mon Sep 17 00:00:00 2001 From: Xingyu Liu Date: Fri, 23 Jan 2026 16:35:55 -0800 Subject: [PATCH 4/8] cleanup Signed-off-by: Xingyu Liu --- vllm/config/model.py | 6 ++---- vllm/config/model_arch.py | 10 +++++++--- .../model_arch_config_convertor.py | 12 ++++++++---- 3 files changed, 17 insertions(+), 11 deletions(-) diff --git a/vllm/config/model.py b/vllm/config/model.py index 5fb4cf0aa19d..a13ebace2180 100644 --- a/vllm/config/model.py +++ b/vllm/config/model.py @@ -1918,7 +1918,7 @@ def _get_and_verify_max_len( ) -> int: """Get and verify the model's maximum length.""" # Get pre-computed derived max model len info (includes RoPE scaling) - max_len_info = model_arch_config.derived_max_model_len_info + max_len_info = model_arch_config.max_model_len_info derived_max_model_len = max_len_info.derived max_len_key = max_len_info.derived_key @@ -1982,9 +1982,7 @@ def _get_and_verify_max_len( # Some models might have a separate key for specifying model_max_length # that will be bigger than derived_max_model_len. We compare user input # with model_max_length and allow this override when it's smaller. - # NOTE: model_max_length is not consolidated into model_arch_config - # as it's primarily used for tokenizer limits, not model architecture. - model_max_length = getattr(hf_config, "model_max_length", None) + model_max_length = max_len_info.model_max_length if model_max_length is None or max_model_len > model_max_length: msg = ( f"User-specified max_model_len ({max_model_len}) is greater " diff --git a/vllm/config/model_arch.py b/vllm/config/model_arch.py index 1394f3efb999..47625e2ab718 100644 --- a/vllm/config/model_arch.py +++ b/vllm/config/model_arch.py @@ -10,8 +10,8 @@ logger = init_logger(__name__) -class DerivedMaxModelLenInfo(NamedTuple): - """Information about the derived maximum model length.""" +class MaxModelLenInfo(NamedTuple): + """Information about the maximum model length.""" derived: float """Maximum supported sequence length after RoPE scaling. @@ -27,6 +27,10 @@ class DerivedMaxModelLenInfo(NamedTuple): Used as the default max_model_len to avoid performance degradation. None for non-LongRoPE models (derived is used instead).""" + model_max_length: int | None + """The model_max_length from hf_config. Used as a fallback for validation + when user-specified max_model_len exceeds derived.""" + @dataclass(config=ConfigDict(arbitrary_types_allowed=True)) class ModelArchitectureConfig: @@ -71,7 +75,7 @@ class ModelArchitectureConfig: is_deepseek_mla: bool """Whether the model is a DeepSeek MLA model.""" - derived_max_model_len_info: DerivedMaxModelLenInfo + max_model_len_info: MaxModelLenInfo """Derived maximum model length information including RoPE scaling.""" # RoPE-related fields diff --git a/vllm/transformers_utils/model_arch_config_convertor.py b/vllm/transformers_utils/model_arch_config_convertor.py index a36fc241d377..da1e9c04838e 100644 --- a/vllm/transformers_utils/model_arch_config_convertor.py +++ b/vllm/transformers_utils/model_arch_config_convertor.py @@ -9,7 +9,7 @@ from vllm import envs from vllm.config.model_arch import ( - DerivedMaxModelLenInfo, + MaxModelLenInfo, ModelArchitectureConfig, ) from vllm.config.utils import getattr_iter @@ -210,7 +210,7 @@ def is_deepseek_mla(self) -> bool: ) return False - def derive_max_model_len_info(self) -> DerivedMaxModelLenInfo: + def derive_max_model_len_info(self) -> MaxModelLenInfo: """Derive maximum model length including RoPE scaling factors. This method computes the derived max model length by: @@ -314,10 +314,14 @@ def derive_max_model_len_info(self) -> DerivedMaxModelLenInfo: else: default_max_model_len = None - return DerivedMaxModelLenInfo( + # Get model_max_length for validation fallback + model_max_length = getattr(self.hf_config, "model_max_length", None) + + return MaxModelLenInfo( derived=derived_max_model_len, derived_key=max_len_key, default=default_max_model_len, + model_max_length=model_max_length, ) def get_uses_mrope(self) -> bool: @@ -340,7 +344,7 @@ def convert(self) -> ModelArchitectureConfig: num_experts=self.get_num_experts(), quantization_config=self.get_quantization_config(), is_deepseek_mla=self.is_deepseek_mla(), - derived_max_model_len_info=self.derive_max_model_len_info(), + max_model_len_info=self.derive_max_model_len_info(), # RoPE-related fields uses_mrope=self.get_uses_mrope(), uses_xdrope_dim=self.get_uses_xdrope_dim(), From 835fe092a31497a1de08e79a31961d6f091b0b4a Mon Sep 17 00:00:00 2001 From: Xingyu Liu Date: Sat, 24 Jan 2026 23:49:25 -0800 Subject: [PATCH 5/8] base model rope gt Signed-off-by: Xingyu Liu --- tests/config/base_model_arch_groundtruth.json | 57 ++++++++++++------- 1 file changed, 38 insertions(+), 19 deletions(-) diff --git a/tests/config/base_model_arch_groundtruth.json b/tests/config/base_model_arch_groundtruth.json index 3401198ad7d5..413895cf1ec8 100644 --- a/tests/config/base_model_arch_groundtruth.json +++ b/tests/config/base_model_arch_groundtruth.json @@ -14,7 +14,8 @@ "num_experts": 0, "is_deepseek_mla": false, "is_multimodal_model": false, - "dtype": "torch.float32" + "dtype": "torch.float32", + "model_max_len": 2048 }, "mistralai/Mamba-Codestral-7B-v0.1": { "architectures": [ @@ -31,7 +32,8 @@ "num_experts": 0, "is_deepseek_mla": false, "is_multimodal_model": false, - "dtype": "torch.bfloat16" + "dtype": "torch.bfloat16", + "model_max_len": 128000 }, "ibm-nasa-geospatial/Prithvi-EO-2.0-300M-TL-Sen1Floods11": { "architectures": [ @@ -48,7 +50,8 @@ "num_experts": 0, "is_deepseek_mla": false, "is_multimodal_model": true, - "dtype": "torch.float32" + "dtype": "torch.float32", + "model_max_len": 2048 }, "tiiuae/falcon-mamba-7b-instruct": { "architectures": [ @@ -65,7 +68,8 @@ "num_experts": 0, "is_deepseek_mla": false, "is_multimodal_model": false, - "dtype": "torch.bfloat16" + "dtype": "torch.bfloat16", + "model_max_len": 2048 }, "Zyphra/Zamba2-7B-instruct": { "architectures": [ @@ -82,7 +86,8 @@ "num_experts": 0, "is_deepseek_mla": false, "is_multimodal_model": false, - "dtype": "torch.bfloat16" + "dtype": "torch.bfloat16", + "model_max_len": 4096 }, "mosaicml/mpt-7b": { "architectures": [ @@ -133,7 +138,8 @@ "num_experts": 0, "is_deepseek_mla": false, "is_multimodal_model": false, - "dtype": "torch.bfloat16" + "dtype": "torch.bfloat16", + "model_max_len": 2048 }, "tiiuae/falcon-40b": { "architectures": [ @@ -150,7 +156,8 @@ "num_experts": 0, "is_deepseek_mla": false, "is_multimodal_model": false, - "dtype": "torch.bfloat16" + "dtype": "torch.bfloat16", + "model_max_len": 2048 }, "luccafong/deepseek_mtp_main_random": { "architectures": [ @@ -167,7 +174,8 @@ "num_experts": 72, "is_deepseek_mla": true, "is_multimodal_model": false, - "dtype": "torch.bfloat16" + "dtype": "torch.bfloat16", + "model_max_len": 163840 }, "luccafong/deepseek_mtp_draft_random": { "architectures": [ @@ -184,7 +192,8 @@ "num_experts": 72, "is_deepseek_mla": true, "is_multimodal_model": false, - "dtype": "torch.bfloat16" + "dtype": "torch.bfloat16", + "model_max_len": 163840 }, "Qwen/Qwen3-Next-80B-A3B-Instruct": { "architectures": [ @@ -201,7 +210,8 @@ "num_experts": 512, "is_deepseek_mla": false, "is_multimodal_model": false, - "dtype": "torch.bfloat16" + "dtype": "torch.bfloat16", + "model_max_len": 262144 }, "tiny-random/qwen3-next-moe": { "architectures": [ @@ -218,7 +228,8 @@ "num_experts": 32, "is_deepseek_mla": false, "is_multimodal_model": false, - "dtype": "torch.bfloat16" + "dtype": "torch.bfloat16", + "model_max_len": 262144 }, "zai-org/GLM-4.5": { "architectures": [ @@ -235,7 +246,8 @@ "num_experts": 160, "is_deepseek_mla": false, "is_multimodal_model": false, - "dtype": "torch.bfloat16" + "dtype": "torch.bfloat16", + "model_max_len": 131072 }, "baidu/ERNIE-4.5-21B-A3B-PT": { "architectures": [ @@ -252,7 +264,8 @@ "num_experts": 64, "is_deepseek_mla": false, "is_multimodal_model": false, - "dtype": "torch.bfloat16" + "dtype": "torch.bfloat16", + "model_max_len": 131072 }, "lmsys/gpt-oss-20b-bf16": { "architectures": [ @@ -269,7 +282,8 @@ "num_experts": 32, "is_deepseek_mla": false, "is_multimodal_model": false, - "dtype": "torch.bfloat16" + "dtype": "torch.bfloat16", + "model_max_len": 131072 }, "deepseek-ai/DeepSeek-V3.2-Exp": { "architectures": [ @@ -286,7 +300,8 @@ "num_experts": 256, "is_deepseek_mla": true, "is_multimodal_model": false, - "dtype": "torch.bfloat16" + "dtype": "torch.bfloat16", + "model_max_len": 163840 }, "meta-llama/Llama-4-Scout-17B-16E-Instruct": { "architectures": [ @@ -303,7 +318,8 @@ "num_experts": 16, "is_deepseek_mla": false, "is_multimodal_model": true, - "dtype": "torch.bfloat16" + "dtype": "torch.bfloat16", + "model_max_len": 10485760 }, "nvidia/Llama-3_3-Nemotron-Super-49B-v1": { "architectures": [ @@ -320,7 +336,8 @@ "num_experts": 0, "is_deepseek_mla": false, "is_multimodal_model": false, - "dtype": "torch.bfloat16" + "dtype": "torch.bfloat16", + "model_max_len": 131072 }, "XiaomiMiMo/MiMo-7B-RL": { "architectures": [ @@ -337,7 +354,8 @@ "num_experts": 0, "is_deepseek_mla": false, "is_multimodal_model": false, - "dtype": "torch.bfloat16" + "dtype": "torch.bfloat16", + "model_max_len": 32768 }, "meituan-longcat/LongCat-Flash-Chat": { "architectures": [ @@ -354,6 +372,7 @@ "num_experts": 512, "is_deepseek_mla": true, "is_multimodal_model": false, - "dtype": "torch.float32" + "dtype": "torch.float32", + "model_max_len": 131072 } } From 5902ecbb026845a1cfd58c055655a641ad956083 Mon Sep 17 00:00:00 2001 From: Xingyu Liu Date: Sat, 24 Jan 2026 23:51:24 -0800 Subject: [PATCH 6/8] add tests Signed-off-by: Xingyu Liu --- tests/config/test_model_arch_config.py | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/tests/config/test_model_arch_config.py b/tests/config/test_model_arch_config.py index 06d4c6e7a865..d711de0b8a5f 100644 --- a/tests/config/test_model_arch_config.py +++ b/tests/config/test_model_arch_config.py @@ -91,11 +91,10 @@ def _assert_model_arch_config( assert model_arch_config.head_size == expected["head_size"] -def _assert_model_config_methods( - model_config, expected: dict, check_head_size: bool = True -): - """Assert model_config methods return expected values.""" +def _assert_model_config(model_config, expected: dict, check_head_size: bool = True): + """Assert model_config return expected values.""" assert model_config.architectures == expected["architectures"] + assert model_config.max_model_len == expected["max_model_len"] assert model_config.get_vocab_size() == expected["vocab_size"] assert model_config.get_hidden_size() == expected["hidden_size"] assert model_config.get_total_num_kv_heads() == expected["total_num_kv_heads"] @@ -120,7 +119,7 @@ def test_base_model_arch_config(model: str): ) _assert_model_arch_config(model_config, expected) - _assert_model_config_methods(model_config, expected) + _assert_model_config(model_config, expected) @pytest.mark.parametrize( @@ -147,6 +146,4 @@ def test_draft_model_arch_config( check_head_size = isinstance(expected["head_size"], int) _assert_model_arch_config(model_config, expected, check_head_size=check_head_size) - _assert_model_config_methods( - model_config, expected, check_head_size=check_head_size - ) + _assert_model_config(model_config, expected, check_head_size=check_head_size) From a3e4f8405852f3e163e2d22e505c0d578950a42b Mon Sep 17 00:00:00 2001 From: Xingyu Liu Date: Sat, 24 Jan 2026 23:59:16 -0800 Subject: [PATCH 7/8] draft gt Signed-off-by: Xingyu Liu --- tests/config/draft_model_arch_groundtruth.json | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/tests/config/draft_model_arch_groundtruth.json b/tests/config/draft_model_arch_groundtruth.json index 5ea8136e9bd9..8b6abadf67c2 100644 --- a/tests/config/draft_model_arch_groundtruth.json +++ b/tests/config/draft_model_arch_groundtruth.json @@ -14,7 +14,8 @@ "num_experts": 0, "is_deepseek_mla": false, "is_multimodal_model": false, - "dtype": "torch.float32" + "dtype": "torch.float32", + "model_max_len": 2048 }, "luccafong/deepseek_mtp_draft_random": { "architectures": [ @@ -31,7 +32,8 @@ "num_experts": 72, "is_deepseek_mla": true, "is_multimodal_model": false, - "dtype": "torch.bfloat16" + "dtype": "torch.bfloat16", + "model_max_len": 163840 }, "eagle618/eagle-deepseek-v3-random": { "architectures": [ @@ -48,7 +50,8 @@ "num_experts": 72, "is_deepseek_mla": true, "is_multimodal_model": false, - "dtype": "bfloat16" + "dtype": "bfloat16", + "model_max_len": 163840 }, "yuhuili/EAGLE-LLaMA3-Instruct-8B": { "architectures": [ @@ -65,7 +68,8 @@ "num_experts": 0, "is_deepseek_mla": false, "is_multimodal_model": false, - "dtype": "float16" + "dtype": "float16", + "model_max_len": 2048 }, "yuhuili/EAGLE3-LLaMA3.1-Instruct-8B": { "architectures": [ @@ -82,6 +86,7 @@ "num_experts": 0, "is_deepseek_mla": false, "is_multimodal_model": false, - "dtype": "float16" + "dtype": "float16", + "model_max_len": 2048 } } From be04dc152e348ae2aac36bf76480d676fc1527ce Mon Sep 17 00:00:00 2001 From: Xingyu Liu Date: Sun, 25 Jan 2026 00:05:55 -0800 Subject: [PATCH 8/8] rename Signed-off-by: Xingyu Liu --- tests/config/base_model_arch_groundtruth.json | 38 +++++++++---------- .../config/draft_model_arch_groundtruth.json | 10 ++--- 2 files changed, 24 insertions(+), 24 deletions(-) diff --git a/tests/config/base_model_arch_groundtruth.json b/tests/config/base_model_arch_groundtruth.json index 413895cf1ec8..aa090e551768 100644 --- a/tests/config/base_model_arch_groundtruth.json +++ b/tests/config/base_model_arch_groundtruth.json @@ -15,7 +15,7 @@ "is_deepseek_mla": false, "is_multimodal_model": false, "dtype": "torch.float32", - "model_max_len": 2048 + "max_model_len": 2048 }, "mistralai/Mamba-Codestral-7B-v0.1": { "architectures": [ @@ -33,7 +33,7 @@ "is_deepseek_mla": false, "is_multimodal_model": false, "dtype": "torch.bfloat16", - "model_max_len": 128000 + "max_model_len": 128000 }, "ibm-nasa-geospatial/Prithvi-EO-2.0-300M-TL-Sen1Floods11": { "architectures": [ @@ -51,7 +51,7 @@ "is_deepseek_mla": false, "is_multimodal_model": true, "dtype": "torch.float32", - "model_max_len": 2048 + "max_model_len": 2048 }, "tiiuae/falcon-mamba-7b-instruct": { "architectures": [ @@ -69,7 +69,7 @@ "is_deepseek_mla": false, "is_multimodal_model": false, "dtype": "torch.bfloat16", - "model_max_len": 2048 + "max_model_len": 2048 }, "Zyphra/Zamba2-7B-instruct": { "architectures": [ @@ -87,7 +87,7 @@ "is_deepseek_mla": false, "is_multimodal_model": false, "dtype": "torch.bfloat16", - "model_max_len": 4096 + "max_model_len": 4096 }, "mosaicml/mpt-7b": { "architectures": [ @@ -139,7 +139,7 @@ "is_deepseek_mla": false, "is_multimodal_model": false, "dtype": "torch.bfloat16", - "model_max_len": 2048 + "max_model_len": 2048 }, "tiiuae/falcon-40b": { "architectures": [ @@ -157,7 +157,7 @@ "is_deepseek_mla": false, "is_multimodal_model": false, "dtype": "torch.bfloat16", - "model_max_len": 2048 + "max_model_len": 2048 }, "luccafong/deepseek_mtp_main_random": { "architectures": [ @@ -175,7 +175,7 @@ "is_deepseek_mla": true, "is_multimodal_model": false, "dtype": "torch.bfloat16", - "model_max_len": 163840 + "max_model_len": 163840 }, "luccafong/deepseek_mtp_draft_random": { "architectures": [ @@ -193,7 +193,7 @@ "is_deepseek_mla": true, "is_multimodal_model": false, "dtype": "torch.bfloat16", - "model_max_len": 163840 + "max_model_len": 163840 }, "Qwen/Qwen3-Next-80B-A3B-Instruct": { "architectures": [ @@ -211,7 +211,7 @@ "is_deepseek_mla": false, "is_multimodal_model": false, "dtype": "torch.bfloat16", - "model_max_len": 262144 + "max_model_len": 262144 }, "tiny-random/qwen3-next-moe": { "architectures": [ @@ -229,7 +229,7 @@ "is_deepseek_mla": false, "is_multimodal_model": false, "dtype": "torch.bfloat16", - "model_max_len": 262144 + "max_model_len": 262144 }, "zai-org/GLM-4.5": { "architectures": [ @@ -247,7 +247,7 @@ "is_deepseek_mla": false, "is_multimodal_model": false, "dtype": "torch.bfloat16", - "model_max_len": 131072 + "max_model_len": 131072 }, "baidu/ERNIE-4.5-21B-A3B-PT": { "architectures": [ @@ -265,7 +265,7 @@ "is_deepseek_mla": false, "is_multimodal_model": false, "dtype": "torch.bfloat16", - "model_max_len": 131072 + "max_model_len": 131072 }, "lmsys/gpt-oss-20b-bf16": { "architectures": [ @@ -283,7 +283,7 @@ "is_deepseek_mla": false, "is_multimodal_model": false, "dtype": "torch.bfloat16", - "model_max_len": 131072 + "max_model_len": 131072 }, "deepseek-ai/DeepSeek-V3.2-Exp": { "architectures": [ @@ -301,7 +301,7 @@ "is_deepseek_mla": true, "is_multimodal_model": false, "dtype": "torch.bfloat16", - "model_max_len": 163840 + "max_model_len": 163840 }, "meta-llama/Llama-4-Scout-17B-16E-Instruct": { "architectures": [ @@ -319,7 +319,7 @@ "is_deepseek_mla": false, "is_multimodal_model": true, "dtype": "torch.bfloat16", - "model_max_len": 10485760 + "max_model_len": 10485760 }, "nvidia/Llama-3_3-Nemotron-Super-49B-v1": { "architectures": [ @@ -337,7 +337,7 @@ "is_deepseek_mla": false, "is_multimodal_model": false, "dtype": "torch.bfloat16", - "model_max_len": 131072 + "max_model_len": 131072 }, "XiaomiMiMo/MiMo-7B-RL": { "architectures": [ @@ -355,7 +355,7 @@ "is_deepseek_mla": false, "is_multimodal_model": false, "dtype": "torch.bfloat16", - "model_max_len": 32768 + "max_model_len": 32768 }, "meituan-longcat/LongCat-Flash-Chat": { "architectures": [ @@ -373,6 +373,6 @@ "is_deepseek_mla": true, "is_multimodal_model": false, "dtype": "torch.float32", - "model_max_len": 131072 + "max_model_len": 131072 } } diff --git a/tests/config/draft_model_arch_groundtruth.json b/tests/config/draft_model_arch_groundtruth.json index 8b6abadf67c2..ad2488b62d40 100644 --- a/tests/config/draft_model_arch_groundtruth.json +++ b/tests/config/draft_model_arch_groundtruth.json @@ -15,7 +15,7 @@ "is_deepseek_mla": false, "is_multimodal_model": false, "dtype": "torch.float32", - "model_max_len": 2048 + "max_model_len": 2048 }, "luccafong/deepseek_mtp_draft_random": { "architectures": [ @@ -33,7 +33,7 @@ "is_deepseek_mla": true, "is_multimodal_model": false, "dtype": "torch.bfloat16", - "model_max_len": 163840 + "max_model_len": 163840 }, "eagle618/eagle-deepseek-v3-random": { "architectures": [ @@ -51,7 +51,7 @@ "is_deepseek_mla": true, "is_multimodal_model": false, "dtype": "bfloat16", - "model_max_len": 163840 + "max_model_len": 163840 }, "yuhuili/EAGLE-LLaMA3-Instruct-8B": { "architectures": [ @@ -69,7 +69,7 @@ "is_deepseek_mla": false, "is_multimodal_model": false, "dtype": "float16", - "model_max_len": 2048 + "max_model_len": 2048 }, "yuhuili/EAGLE3-LLaMA3.1-Instruct-8B": { "architectures": [ @@ -87,6 +87,6 @@ "is_deepseek_mla": false, "is_multimodal_model": false, "dtype": "float16", - "model_max_len": 2048 + "max_model_len": 2048 } }