diff --git a/tests/config/base_model_arch_groundtruth.json b/tests/config/base_model_arch_groundtruth.json index 3401198ad7d5..aa090e551768 100644 --- a/tests/config/base_model_arch_groundtruth.json +++ b/tests/config/base_model_arch_groundtruth.json @@ -14,7 +14,8 @@ "num_experts": 0, "is_deepseek_mla": false, "is_multimodal_model": false, - "dtype": "torch.float32" + "dtype": "torch.float32", + "max_model_len": 2048 }, "mistralai/Mamba-Codestral-7B-v0.1": { "architectures": [ @@ -31,7 +32,8 @@ "num_experts": 0, "is_deepseek_mla": false, "is_multimodal_model": false, - "dtype": "torch.bfloat16" + "dtype": "torch.bfloat16", + "max_model_len": 128000 }, "ibm-nasa-geospatial/Prithvi-EO-2.0-300M-TL-Sen1Floods11": { "architectures": [ @@ -48,7 +50,8 @@ "num_experts": 0, "is_deepseek_mla": false, "is_multimodal_model": true, - "dtype": "torch.float32" + "dtype": "torch.float32", + "max_model_len": 2048 }, "tiiuae/falcon-mamba-7b-instruct": { "architectures": [ @@ -65,7 +68,8 @@ "num_experts": 0, "is_deepseek_mla": false, "is_multimodal_model": false, - "dtype": "torch.bfloat16" + "dtype": "torch.bfloat16", + "max_model_len": 2048 }, "Zyphra/Zamba2-7B-instruct": { "architectures": [ @@ -82,7 +86,8 @@ "num_experts": 0, "is_deepseek_mla": false, "is_multimodal_model": false, - "dtype": "torch.bfloat16" + "dtype": "torch.bfloat16", + "max_model_len": 4096 }, "mosaicml/mpt-7b": { "architectures": [ @@ -133,7 +138,8 @@ "num_experts": 0, "is_deepseek_mla": false, "is_multimodal_model": false, - "dtype": "torch.bfloat16" + "dtype": "torch.bfloat16", + "max_model_len": 2048 }, "tiiuae/falcon-40b": { "architectures": [ @@ -150,7 +156,8 @@ "num_experts": 0, "is_deepseek_mla": false, "is_multimodal_model": false, - "dtype": "torch.bfloat16" + "dtype": "torch.bfloat16", + "max_model_len": 2048 }, "luccafong/deepseek_mtp_main_random": { "architectures": [ @@ -167,7 +174,8 @@ "num_experts": 72, "is_deepseek_mla": true, "is_multimodal_model": false, - "dtype": "torch.bfloat16" + "dtype": "torch.bfloat16", + "max_model_len": 163840 }, "luccafong/deepseek_mtp_draft_random": { "architectures": [ @@ -184,7 +192,8 @@ "num_experts": 72, "is_deepseek_mla": true, "is_multimodal_model": false, - "dtype": "torch.bfloat16" + "dtype": "torch.bfloat16", + "max_model_len": 163840 }, "Qwen/Qwen3-Next-80B-A3B-Instruct": { "architectures": [ @@ -201,7 +210,8 @@ "num_experts": 512, "is_deepseek_mla": false, "is_multimodal_model": false, - "dtype": "torch.bfloat16" + "dtype": "torch.bfloat16", + "max_model_len": 262144 }, "tiny-random/qwen3-next-moe": { "architectures": [ @@ -218,7 +228,8 @@ "num_experts": 32, "is_deepseek_mla": false, "is_multimodal_model": false, - "dtype": "torch.bfloat16" + "dtype": "torch.bfloat16", + "max_model_len": 262144 }, "zai-org/GLM-4.5": { "architectures": [ @@ -235,7 +246,8 @@ "num_experts": 160, "is_deepseek_mla": false, "is_multimodal_model": false, - "dtype": "torch.bfloat16" + "dtype": "torch.bfloat16", + "max_model_len": 131072 }, "baidu/ERNIE-4.5-21B-A3B-PT": { "architectures": [ @@ -252,7 +264,8 @@ "num_experts": 64, "is_deepseek_mla": false, "is_multimodal_model": false, - "dtype": "torch.bfloat16" + "dtype": "torch.bfloat16", + "max_model_len": 131072 }, "lmsys/gpt-oss-20b-bf16": { "architectures": [ @@ -269,7 +282,8 @@ "num_experts": 32, "is_deepseek_mla": false, "is_multimodal_model": false, - "dtype": "torch.bfloat16" + "dtype": "torch.bfloat16", + "max_model_len": 131072 }, "deepseek-ai/DeepSeek-V3.2-Exp": { "architectures": [ @@ -286,7 +300,8 @@ "num_experts": 256, "is_deepseek_mla": true, "is_multimodal_model": false, - "dtype": "torch.bfloat16" + "dtype": "torch.bfloat16", + "max_model_len": 163840 }, "meta-llama/Llama-4-Scout-17B-16E-Instruct": { "architectures": [ @@ -303,7 +318,8 @@ "num_experts": 16, "is_deepseek_mla": false, "is_multimodal_model": true, - "dtype": "torch.bfloat16" + "dtype": "torch.bfloat16", + "max_model_len": 10485760 }, "nvidia/Llama-3_3-Nemotron-Super-49B-v1": { "architectures": [ @@ -320,7 +336,8 @@ "num_experts": 0, "is_deepseek_mla": false, "is_multimodal_model": false, - "dtype": "torch.bfloat16" + "dtype": "torch.bfloat16", + "max_model_len": 131072 }, "XiaomiMiMo/MiMo-7B-RL": { "architectures": [ @@ -337,7 +354,8 @@ "num_experts": 0, "is_deepseek_mla": false, "is_multimodal_model": false, - "dtype": "torch.bfloat16" + "dtype": "torch.bfloat16", + "max_model_len": 32768 }, "meituan-longcat/LongCat-Flash-Chat": { "architectures": [ @@ -354,6 +372,7 @@ "num_experts": 512, "is_deepseek_mla": true, "is_multimodal_model": false, - "dtype": "torch.float32" + "dtype": "torch.float32", + "max_model_len": 131072 } } diff --git a/tests/config/draft_model_arch_groundtruth.json b/tests/config/draft_model_arch_groundtruth.json index 5ea8136e9bd9..ad2488b62d40 100644 --- a/tests/config/draft_model_arch_groundtruth.json +++ b/tests/config/draft_model_arch_groundtruth.json @@ -14,7 +14,8 @@ "num_experts": 0, "is_deepseek_mla": false, "is_multimodal_model": false, - "dtype": "torch.float32" + "dtype": "torch.float32", + "max_model_len": 2048 }, "luccafong/deepseek_mtp_draft_random": { "architectures": [ @@ -31,7 +32,8 @@ "num_experts": 72, "is_deepseek_mla": true, "is_multimodal_model": false, - "dtype": "torch.bfloat16" + "dtype": "torch.bfloat16", + "max_model_len": 163840 }, "eagle618/eagle-deepseek-v3-random": { "architectures": [ @@ -48,7 +50,8 @@ "num_experts": 72, "is_deepseek_mla": true, "is_multimodal_model": false, - "dtype": "bfloat16" + "dtype": "bfloat16", + "max_model_len": 163840 }, "yuhuili/EAGLE-LLaMA3-Instruct-8B": { "architectures": [ @@ -65,7 +68,8 @@ "num_experts": 0, "is_deepseek_mla": false, "is_multimodal_model": false, - "dtype": "float16" + "dtype": "float16", + "max_model_len": 2048 }, "yuhuili/EAGLE3-LLaMA3.1-Instruct-8B": { "architectures": [ @@ -82,6 +86,7 @@ "num_experts": 0, "is_deepseek_mla": false, "is_multimodal_model": false, - "dtype": "float16" + "dtype": "float16", + "max_model_len": 2048 } } diff --git a/tests/config/test_model_arch_config.py b/tests/config/test_model_arch_config.py index 06d4c6e7a865..d711de0b8a5f 100644 --- a/tests/config/test_model_arch_config.py +++ b/tests/config/test_model_arch_config.py @@ -91,11 +91,10 @@ def _assert_model_arch_config( assert model_arch_config.head_size == expected["head_size"] -def _assert_model_config_methods( - model_config, expected: dict, check_head_size: bool = True -): - """Assert model_config methods return expected values.""" +def _assert_model_config(model_config, expected: dict, check_head_size: bool = True): + """Assert model_config return expected values.""" assert model_config.architectures == expected["architectures"] + assert model_config.max_model_len == expected["max_model_len"] assert model_config.get_vocab_size() == expected["vocab_size"] assert model_config.get_hidden_size() == expected["hidden_size"] assert model_config.get_total_num_kv_heads() == expected["total_num_kv_heads"] @@ -120,7 +119,7 @@ def test_base_model_arch_config(model: str): ) _assert_model_arch_config(model_config, expected) - _assert_model_config_methods(model_config, expected) + _assert_model_config(model_config, expected) @pytest.mark.parametrize( @@ -147,6 +146,4 @@ def test_draft_model_arch_config( check_head_size = isinstance(expected["head_size"], int) _assert_model_arch_config(model_config, expected, check_head_size=check_head_size) - _assert_model_config_methods( - model_config, expected, check_head_size=check_head_size - ) + _assert_model_config(model_config, expected, check_head_size=check_head_size) diff --git a/vllm/config/model.py b/vllm/config/model.py index 883e0b17ef0e..a13ebace2180 100644 --- a/vllm/config/model.py +++ b/vllm/config/model.py @@ -29,12 +29,9 @@ get_pooling_config, get_sentence_transformer_tokenizer_config, is_encoder_decoder, - is_rope_parameters_nested, try_get_dense_modules, try_get_generation_config, try_get_tokenizer_config, - uses_mrope, - uses_xdrope_dim, ) from vllm.transformers_utils.gguf_utils import ( is_gguf, @@ -1407,11 +1404,11 @@ def uses_alibi(self) -> bool: @property def uses_mrope(self) -> bool: - return uses_mrope(self.hf_config) + return self.model_arch_config.uses_mrope @property def uses_xdrope_dim(self) -> int: - return uses_xdrope_dim(self.hf_config) + return self.model_arch_config.uses_xdrope_dim @property def is_multimodal_model(self) -> bool: @@ -1920,9 +1917,10 @@ def _get_and_verify_max_len( encoder_config: Any | None = None, ) -> int: """Get and verify the model's maximum length.""" - (derived_max_model_len, max_len_key) = ( - model_arch_config.derived_max_model_len_and_key - ) + # Get pre-computed derived max model len info (includes RoPE scaling) + max_len_info = model_arch_config.max_model_len_info + derived_max_model_len = max_len_info.derived + max_len_key = max_len_info.derived_key # If sliding window is manually disabled, max_length should be less # than the sliding window length in the model config. @@ -1962,31 +1960,6 @@ def _get_and_verify_max_len( ) derived_max_model_len = default_max_len - # In Transformers v5 rope_parameters could be TypedDict or dict[str, TypedDict]. - # To simplify the verification, we convert it to dict[str, TypedDict]. - rope_parameters = getattr(hf_config, "rope_parameters", None) - if rope_parameters and not is_rope_parameters_nested(rope_parameters): - rope_parameters = {"": rope_parameters} - - # NOTE(woosuk): Gemma3's max_model_len (128K) is already scaled by RoPE - # scaling, so we skip applying the scaling factor again. - if rope_parameters is not None and "gemma3" not in hf_config.model_type: - scaling_factor = 1.0 - for rp in rope_parameters.values(): - # No need to consider "type" key because of patch_rope_parameters when - # loading HF config - rope_type = rp["rope_type"] - - if rope_type not in ("su", "longrope", "llama3"): - # NOTE: rope_type == "default" does not define factor https://github.com/huggingface/transformers/blob/v4.45.2/src/transformers/modeling_rope_utils.py - # NOTE: This assumes all layer types have the same scaling factor. - scaling_factor = rp.get("factor", scaling_factor) - - if rope_type == "yarn": - derived_max_model_len = rp["original_max_position_embeddings"] - # Do this outside loop since all layer types should have the same scaling - derived_max_model_len *= scaling_factor - if encoder_config and "max_seq_length" in encoder_config: derived_max_model_len = encoder_config["max_seq_length"] @@ -1994,17 +1967,12 @@ def _get_and_verify_max_len( # then use that derived from the model config as a default value. # When -1 is specified, the engine will later auto-fit to available memory. if max_model_len is None or max_model_len == -1: - # For LongRoPE, default to original_max_position_embeddings to avoid - # performance degradation for shorter sequences - if rope_parameters is not None and any( - rp["rope_type"] == "longrope" for rp in rope_parameters.values() - ): - max_model_len = int( - getattr( - hf_config, "original_max_position_embeddings", derived_max_model_len - ) - ) + if max_len_info.default is not None: + # For LongRoPE, default to original_max_position_embeddings to avoid + # performance degradation for shorter sequences + max_model_len = int(max_len_info.default) else: + # Non-LongRoPE: use derived_max_model_len with caps applied max_model_len = int(derived_max_model_len) max_model_len = current_platform.check_max_model_len(max_model_len) @@ -2014,7 +1982,7 @@ def _get_and_verify_max_len( # Some models might have a separate key for specifying model_max_length # that will be bigger than derived_max_model_len. We compare user input # with model_max_length and allow this override when it's smaller. - model_max_length = getattr(hf_config, "model_max_length", None) + model_max_length = max_len_info.model_max_length if model_max_length is None or max_model_len > model_max_length: msg = ( f"User-specified max_model_len ({max_model_len}) is greater " diff --git a/vllm/config/model_arch.py b/vllm/config/model_arch.py index d55e2a3399b3..47625e2ab718 100644 --- a/vllm/config/model_arch.py +++ b/vllm/config/model_arch.py @@ -1,6 +1,6 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project -from typing import Any +from typing import Any, NamedTuple from pydantic import ConfigDict from pydantic.dataclasses import dataclass @@ -10,6 +10,28 @@ logger = init_logger(__name__) +class MaxModelLenInfo(NamedTuple): + """Information about the maximum model length.""" + + derived: float + """Maximum supported sequence length after RoPE scaling. + Used for: + 1. Validation - user-specified max_model_len cannot exceed this. + 2. Default for non-LongRoPE models (with sliding_window/tokenizer caps).""" + + derived_key: str | None + """The config key used to derive the max length (for error messages).""" + + default: float | None + """For LongRoPE models only: original_max_position_embeddings. + Used as the default max_model_len to avoid performance degradation. + None for non-LongRoPE models (derived is used instead).""" + + model_max_length: int | None + """The model_max_length from hf_config. Used as a fallback for validation + when user-specified max_model_len exceeds derived.""" + + @dataclass(config=ConfigDict(arbitrary_types_allowed=True)) class ModelArchitectureConfig: """ @@ -53,5 +75,12 @@ class ModelArchitectureConfig: is_deepseek_mla: bool """Whether the model is a DeepSeek MLA model.""" - derived_max_model_len_and_key: tuple[float, str | None] - """Derived maximum model length and key from the hf config.""" + max_model_len_info: MaxModelLenInfo + """Derived maximum model length information including RoPE scaling.""" + + # RoPE-related fields + uses_mrope: bool + """Whether the model uses M-RoPE (multi-dimensional rotary position embedding).""" + + uses_xdrope_dim: int + """Number of dimensions for XD-RoPE. 0 if not used.""" diff --git a/vllm/transformers_utils/model_arch_config_convertor.py b/vllm/transformers_utils/model_arch_config_convertor.py index 6df4bb64dceb..da1e9c04838e 100644 --- a/vllm/transformers_utils/model_arch_config_convertor.py +++ b/vllm/transformers_utils/model_arch_config_convertor.py @@ -1,7 +1,7 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project -from typing import final +from typing import Any, final import torch from safetensors.torch import _TYPES as _SAFETENSORS_TO_TORCH_DTYPE @@ -9,12 +9,16 @@ from vllm import envs from vllm.config.model_arch import ( + MaxModelLenInfo, ModelArchitectureConfig, ) from vllm.config.utils import getattr_iter from vllm.logger import init_logger from vllm.transformers_utils.config import ( + is_rope_parameters_nested, try_get_safetensors_metadata, + uses_mrope, + uses_xdrope_dim, ) from vllm.utils.torch_utils import common_broadcastable_dtype @@ -206,8 +210,15 @@ def is_deepseek_mla(self) -> bool: ) return False - def derive_max_model_len_and_key(self) -> tuple[float, str | None]: - derived_max_model_len = float("inf") + def derive_max_model_len_info(self) -> MaxModelLenInfo: + """Derive maximum model length including RoPE scaling factors. + + This method computes the derived max model length by: + 1. Finding the base max length from various config keys + 2. Applying RoPE scaling factors (linear, dynamic, yarn, etc.) + 3. Detecting LongRoPE for special handling + """ + derived_max_model_len: float = float("inf") possible_keys = [ # OPT "max_position_embeddings", @@ -227,7 +238,7 @@ def derive_max_model_len_and_key(self) -> tuple[float, str | None]: "seq_len", ] # Choose the smallest "max_length" from the possible keys - max_len_key = None + max_len_key: str | None = None for key in possible_keys: max_len = getattr(self.hf_text_config, key, None) if max_len is not None: @@ -239,7 +250,85 @@ def derive_max_model_len_and_key(self) -> tuple[float, str | None]: if tmp_max_len := getattr(self.hf_text_config, "model_max_length", None): max_len_key = "model_max_length" derived_max_model_len = tmp_max_len - return derived_max_model_len, max_len_key + + # Get rope_parameters and apply RoPE scaling + rope_parameters = getattr(self.hf_text_config, "rope_parameters", None) + is_longrope = False + original_max_position_embeddings: int | None = None + + # Get original_max_position_embeddings from config or rope_parameters + original_max_position_embeddings = getattr( + self.hf_text_config, "original_max_position_embeddings", None + ) + if original_max_position_embeddings is None and rope_parameters is not None: + original_max_position_embeddings = rope_parameters.get( + "original_max_position_embeddings" + ) + + if rope_parameters is not None: + # In Transformers v5 rope_parameters could be TypedDict or + # dict[str, TypedDict]. Normalize to dict[str, TypedDict]. + if not is_rope_parameters_nested(rope_parameters): + rope_params_dict: dict[str, dict[str, Any]] = {"": rope_parameters} + else: + rope_params_dict = rope_parameters + + # Check if any layer uses longrope + is_longrope = any( + rp.get("rope_type") == "longrope" for rp in rope_params_dict.values() + ) + + # NOTE(woosuk): Gemma3's max_model_len (128K) is already scaled by RoPE + # scaling, so we skip applying the scaling factor again. + model_type = getattr(self.hf_config, "model_type", "") + if "gemma3" not in model_type: + scaling_factor = 1.0 + for rp in rope_params_dict.values(): + rope_type = rp.get("rope_type", "default") + + if rope_type not in ("su", "longrope", "llama3"): + # NOTE: rope_type == "default" does not define factor + # NOTE: This assumes all layer types have the same + # scaling factor. + scaling_factor = rp.get("factor", scaling_factor) + + if rope_type == "yarn": + # For yarn, use original_max_position_embeddings + # from rope_parameters + yarn_ompe = rp.get("original_max_position_embeddings") + if yarn_ompe is not None: + derived_max_model_len = yarn_ompe + + # Apply scaling factor outside the loop since all layer types + # should have the same scaling + derived_max_model_len *= scaling_factor + + # Compute default_max_model_len: + # For LongRoPE, use original_max_position_embeddings to avoid + # performance degradation for shorter sequences. + # For non-LongRoPE, return None (caller should use derived_max_model_len). + if is_longrope and original_max_position_embeddings is not None: + default_max_model_len: float | None = float( + original_max_position_embeddings + ) + else: + default_max_model_len = None + + # Get model_max_length for validation fallback + model_max_length = getattr(self.hf_config, "model_max_length", None) + + return MaxModelLenInfo( + derived=derived_max_model_len, + derived_key=max_len_key, + default=default_max_model_len, + model_max_length=model_max_length, + ) + + def get_uses_mrope(self) -> bool: + return uses_mrope(self.hf_config) + + def get_uses_xdrope_dim(self) -> int: + return uses_xdrope_dim(self.hf_config) def convert(self) -> ModelArchitectureConfig: model_arch_config = ModelArchitectureConfig( @@ -255,7 +344,10 @@ def convert(self) -> ModelArchitectureConfig: num_experts=self.get_num_experts(), quantization_config=self.get_quantization_config(), is_deepseek_mla=self.is_deepseek_mla(), - derived_max_model_len_and_key=self.derive_max_model_len_and_key(), + max_model_len_info=self.derive_max_model_len_info(), + # RoPE-related fields + uses_mrope=self.get_uses_mrope(), + uses_xdrope_dim=self.get_uses_xdrope_dim(), ) return model_arch_config