diff --git a/tools/pre_commit/mypy.py b/tools/pre_commit/mypy.py index a3aa54634725..56272df51e1b 100755 --- a/tools/pre_commit/mypy.py +++ b/tools/pre_commit/mypy.py @@ -26,6 +26,8 @@ FILES = [ "vllm/*.py", "vllm/assets", + "vllm/attention", + "vllm/compilation", "vllm/distributed", "vllm/entrypoints", "vllm/executor", @@ -42,8 +44,6 @@ # from "skip" to "silent", move the following directories to FILES SEPARATE_GROUPS = [ "tests", - "vllm/attention", - "vllm/compilation", "vllm/engine", "vllm/inputs", "vllm/lora", diff --git a/vllm/attention/layer.py b/vllm/attention/layer.py index a3444c1ac82c..b331abc3c365 100644 --- a/vllm/attention/layer.py +++ b/vllm/attention/layer.py @@ -113,7 +113,9 @@ def maybe_get_vit_flash_attn_backend( if use_upstream_fa: from flash_attn import flash_attn_varlen_func else: - from vllm.vllm_flash_attn import flash_attn_varlen_func + from vllm.vllm_flash_attn import ( # type: ignore[attr-defined] + flash_attn_varlen_func, + ) else: flash_attn_varlen_func = None @@ -154,6 +156,7 @@ def __init__( `self.kv_cache`. """ super().__init__() + sliding_window: int | None if per_layer_sliding_window is not None: # per-layer sliding window sliding_window = per_layer_sliding_window @@ -407,7 +410,7 @@ def process_weights_after_loading(self, act_dtype: torch.dtype): def get_attn_backend(self) -> type[AttentionBackend]: return self.attn_backend - def get_kv_cache_spec(self, vllm_config: VllmConfig) -> KVCacheSpec: + def get_kv_cache_spec(self, vllm_config: VllmConfig) -> KVCacheSpec | None: # Block size may get updated after model loading, refresh it block_size = vllm_config.cache_config.block_size # Should not be called for enc-dec or encoder-only attention. diff --git a/vllm/attention/layers/chunked_local_attention.py b/vllm/attention/layers/chunked_local_attention.py index 18422404d08f..b2a8af396ecc 100644 --- a/vllm/attention/layers/chunked_local_attention.py +++ b/vllm/attention/layers/chunked_local_attention.py @@ -1,7 +1,8 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project import functools -from typing import ClassVar +from collections.abc import Hashable +from typing import ClassVar, cast import torch @@ -24,7 +25,7 @@ @functools.lru_cache def create_chunked_local_attention_backend( - underlying_attn_backend: AttentionBackend, + underlying_attn_backend: type[AttentionBackend], attention_chunk_size: int, block_size: int, ) -> type[AttentionBackend]: @@ -84,7 +85,9 @@ def __init__( ) attn_backend = create_chunked_local_attention_backend( - underlying_attn_backend, attention_chunk_size, block_size + cast(Hashable, underlying_attn_backend), + attention_chunk_size, + block_size, ) else: # in v0 the local attention is handled inside the backends diff --git a/vllm/attention/layers/cross_attention.py b/vllm/attention/layers/cross_attention.py index a40a66308a66..bc34c2bf08fe 100644 --- a/vllm/attention/layers/cross_attention.py +++ b/vllm/attention/layers/cross_attention.py @@ -1,7 +1,9 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project import functools +from collections.abc import Hashable from copy import copy +from typing import cast import numpy as np import torch @@ -80,7 +82,7 @@ def _get_cross_slot_mapping( @functools.lru_cache def create_cross_attention_backend( - underlying_attn_backend: AttentionBackend, + underlying_attn_backend: type[AttentionBackend], ) -> type[AttentionBackend]: prefix = "CrossAttention_" underlying_builder = underlying_attn_backend.get_builder_cls() @@ -155,7 +157,9 @@ def __init__( head_size, dtype, kv_cache_dtype, block_size ) - attn_backend = create_cross_attention_backend(underlying_attn_backend) + attn_backend = create_cross_attention_backend( + cast(Hashable, underlying_attn_backend) + ) else: # in v0 cross attention is handled inside the backends attn_backend = None diff --git a/vllm/attention/layers/encoder_only_attention.py b/vllm/attention/layers/encoder_only_attention.py index 8d2a046757fe..0ac838df2854 100644 --- a/vllm/attention/layers/encoder_only_attention.py +++ b/vllm/attention/layers/encoder_only_attention.py @@ -1,7 +1,9 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project import functools +from collections.abc import Hashable from copy import copy +from typing import cast import torch @@ -19,12 +21,11 @@ CommonAttentionMetadata, subclass_attention_backend, ) -from vllm.v1.kv_cache_interface import KVCacheSpec @functools.lru_cache def create_encoder_only_attention_backend( - underlying_attn_backend: AttentionBackend, + underlying_attn_backend: type[AttentionBackend], ) -> type[AttentionBackend]: prefix = "EncoderOnlyAttention_" underlying_builder = underlying_attn_backend.get_builder_cls() @@ -80,7 +81,7 @@ def __init__( ) attn_backend = create_encoder_only_attention_backend( - underlying_attn_backend + cast(Hashable, underlying_attn_backend) ) else: # in v0 encoder only attention is handled inside the backends @@ -101,6 +102,6 @@ def __init__( **kwargs, ) - def get_kv_cache_spec(self, vllm_config: VllmConfig) -> KVCacheSpec: + def get_kv_cache_spec(self, vllm_config: VllmConfig) -> None: # Does not need KV cache return None diff --git a/vllm/attention/selector.py b/vllm/attention/selector.py index 9890d8d80cba..ad1001981f74 100644 --- a/vllm/attention/selector.py +++ b/vllm/attention/selector.py @@ -166,11 +166,8 @@ def _cached_get_attn_backend( # # THIS SELECTION OVERRIDES THE VLLM_ATTENTION_BACKEND # ENVIRONMENT VARIABLE. - selected_backend = None - backend_by_global_setting: _Backend | None = get_global_forced_attn_backend() - if backend_by_global_setting is not None: - selected_backend = backend_by_global_setting - else: + selected_backend = get_global_forced_attn_backend() + if selected_backend is None: # Check the environment variable and override if specified backend_by_env_var: str | None = envs.VLLM_ATTENTION_BACKEND if backend_by_env_var is not None: diff --git a/vllm/attention/utils/fa_utils.py b/vllm/attention/utils/fa_utils.py index b92b822c1d19..1df791ea8089 100644 --- a/vllm/attention/utils/fa_utils.py +++ b/vllm/attention/utils/fa_utils.py @@ -8,22 +8,22 @@ logger = init_logger(__name__) if current_platform.is_cuda(): - from vllm import _custom_ops as ops + from vllm import _custom_ops - reshape_and_cache_flash = ops.reshape_and_cache_flash - from vllm.vllm_flash_attn import flash_attn_varlen_func, get_scheduler_metadata + reshape_and_cache_flash = _custom_ops.reshape_and_cache_flash + from vllm.vllm_flash_attn import ( # type: ignore[attr-defined] + flash_attn_varlen_func, + get_scheduler_metadata, + ) elif current_platform.is_xpu(): - from vllm._ipex_ops import ipex_ops as ops + from vllm._ipex_ops import ipex_ops - reshape_and_cache_flash = ops.reshape_and_cache_flash - flash_attn_varlen_func = ops.flash_attn_varlen_func - get_scheduler_metadata = ops.get_scheduler_metadata + reshape_and_cache_flash = ipex_ops.reshape_and_cache_flash + flash_attn_varlen_func = ipex_ops.flash_attn_varlen_func # type: ignore[assignment] + get_scheduler_metadata = ipex_ops.get_scheduler_metadata def get_flash_attn_version(requires_alibi: bool = False) -> int | None: - # import here to avoid circular dependencies - from vllm.platforms import current_platform - if current_platform.is_xpu(): return 2 try: @@ -74,24 +74,26 @@ def get_flash_attn_version(requires_alibi: bool = False) -> int | None: def flash_attn_supports_fp8() -> bool: + device_capability = current_platform.get_device_capability() return ( get_flash_attn_version() == 3 - and current_platform.get_device_capability().major == 9 + and device_capability is not None + and device_capability.major == 9 ) -def flash_attn_supports_mla(): - from vllm.platforms import current_platform - +def flash_attn_supports_mla() -> bool: if current_platform.is_cuda(): try: from vllm.vllm_flash_attn.flash_attn_interface import ( is_fa_version_supported, ) + device_capability = current_platform.get_device_capability() return ( is_fa_version_supported(3) - and current_platform.get_device_capability()[0] == 9 + and device_capability is not None + and device_capability.major == 9 ) except (ImportError, AssertionError): pass diff --git a/vllm/compilation/collective_fusion.py b/vllm/compilation/collective_fusion.py index 7294ddce64ba..295abd72d4f9 100644 --- a/vllm/compilation/collective_fusion.py +++ b/vllm/compilation/collective_fusion.py @@ -50,7 +50,7 @@ class BasePattern: - def __init__(self, dtype: torch.dtype, device: str): + def __init__(self, dtype: torch.dtype, device: torch.types.Device): self.dtype = dtype self.device = device self.tp = get_tp_group() @@ -664,7 +664,7 @@ def __init__( self, epsilon: float, dtype: torch.dtype, - device: str, + device: torch.types.Device, allreduce_params: FlashInferFusedAllReduceParams, ): super().__init__(dtype, device) @@ -719,7 +719,7 @@ def __init__( self, epsilon: float, dtype: torch.dtype, - device: str, + device: torch.types.Device, allreduce_params: FlashInferFusedAllReduceParams, ): super().__init__(dtype, device) @@ -786,7 +786,7 @@ def __init__( self, epsilon: float, dtype: torch.dtype, - device: str, + device: torch.types.Device, allreduce_params: FlashInferFusedAllReduceParams, ): super().__init__(dtype, device) @@ -855,7 +855,7 @@ def __init__( self, epsilon: float, dtype: torch.dtype, - device: str, + device: torch.types.Device, allreduce_params: FlashInferFusedAllReduceParams, ): super().__init__(dtype, device) @@ -929,7 +929,7 @@ def __init__( self, epsilon: float, dtype: torch.dtype, - device: str, + device: torch.types.Device, allreduce_params: FlashInferFusedAllReduceParams, ): super().__init__(dtype, device) @@ -1015,7 +1015,7 @@ def __init__( self, epsilon: float, dtype: torch.dtype, - device: str, + device: torch.types.Device, allreduce_params: FlashInferFusedAllReduceParams, ): super().__init__(dtype, device) diff --git a/vllm/compilation/cuda_graph.py b/vllm/compilation/cuda_graph.py index a2e0abfebc2c..fe8e8b33dac2 100644 --- a/vllm/compilation/cuda_graph.py +++ b/vllm/compilation/cuda_graph.py @@ -125,6 +125,7 @@ def __call__(self, *args, **kwargs): # runtime modes. return self.runnable(*args, **kwargs) + assert batch_descriptor is not None if batch_descriptor not in self.concrete_cudagraph_entries: # create a new entry for this batch descriptor self.concrete_cudagraph_entries[batch_descriptor] = CUDAGraphEntry( diff --git a/vllm/compilation/sequence_parallelism.py b/vllm/compilation/sequence_parallelism.py index 31624a8fdcc0..c4a4b53f87e7 100644 --- a/vllm/compilation/sequence_parallelism.py +++ b/vllm/compilation/sequence_parallelism.py @@ -108,7 +108,7 @@ def __init__( self, epsilon: float, dtype: torch.dtype, - device: str, + device: torch.types.Device, quant_op: torch._ops.OpOverload | None = None, **kwargs, ): @@ -257,7 +257,11 @@ def replacement( class FirstAllReduceRMSNormStaticFP8Pattern(_SequenceParallelPatternHelper): def __init__( - self, epsilon: float, dtype: torch.dtype, device: str, op: torch._ops.OpOverload + self, + epsilon: float, + dtype: torch.dtype, + device: torch.types.Device, + op: torch._ops.OpOverload, ): super().__init__(epsilon, dtype, device, quant_op=op) @@ -313,7 +317,11 @@ def replacement( class MiddleAllReduceRMSNormStaticFP8Pattern(_SequenceParallelPatternHelper): def __init__( - self, epsilon: float, dtype: torch.dtype, device: str, op: torch._ops.OpOverload + self, + epsilon: float, + dtype: torch.dtype, + device: torch.types.Device, + op: torch._ops.OpOverload, ): super().__init__(epsilon, dtype, device, quant_op=op) @@ -373,7 +381,11 @@ def replacement( class LastAllReduceRMSNormStaticFP8Pattern(_SequenceParallelPatternHelper): def __init__( - self, epsilon: float, dtype: torch.dtype, device: str, op: torch._ops.OpOverload + self, + epsilon: float, + dtype: torch.dtype, + device: torch.types.Device, + op: torch._ops.OpOverload, ): super().__init__(epsilon, dtype, device, quant_op=op) diff --git a/vllm/compilation/vllm_inductor_pass.py b/vllm/compilation/vllm_inductor_pass.py index 08721e3ae4a2..3e999e37c4a0 100644 --- a/vllm/compilation/vllm_inductor_pass.py +++ b/vllm/compilation/vllm_inductor_pass.py @@ -43,7 +43,7 @@ def __init__(self, config: VllmConfig): ) self.pass_config = config.compilation_config.pass_config self.model_dtype = config.model_config.dtype if config.model_config else None - self.device = config.device_config.device if config.device_config else None + self.device = config.device_config.device self.pass_name = self.__class__.__name__ @staticmethod diff --git a/vllm/config/compilation.py b/vllm/config/compilation.py index 61e73414335a..480aebe58678 100644 --- a/vllm/config/compilation.py +++ b/vllm/config/compilation.py @@ -9,7 +9,7 @@ from pathlib import Path from typing import TYPE_CHECKING, Any, ClassVar -from pydantic import TypeAdapter, field_validator +from pydantic import Field, TypeAdapter, field_validator from pydantic.dataclasses import dataclass from vllm.compilation.inductor_pass import CallableInductorPass, InductorPass @@ -179,25 +179,24 @@ class CompilationConfig: """ # Top-level Compilation control - level: int | None = None + level: int = Field(default=None, ge=0, le=3) """ Level is deprecated and will be removed in the next release, either 0.12.0 or 0.11.2 whichever is soonest. Please use mode. Currently all levels are mapped to mode. """ # Top-level Compilation control - mode: int | None = None - """The compilation approach used for torch.compile-based compilation of the - model. + mode: int = Field(default=None, ge=0, le=3) + """ + The compilation approach used for torch.compile-based compilation of the model. + If unset, we will select the default compilation mode. For V1 engine this is 3. - - None: If None, we will select the default compilation mode. - For V1 engine this is 3. - 0: NONE: No torch.compile compilation is applied, model runs in fully - eager pytorch mode. The model runs as-is. - - 1: STOCK_TORCH_COMPILE: The standard `torch.compile` compilation pipeline. + eager pytorch mode. The model runs as-is.\n + - 1: STOCK_TORCH_COMPILE: The standard `torch.compile` compilation pipeline.\n - 2: DYNAMO_TRACE_ONCE: Single Dynamo trace through the model, avoiding recompilation by removing guards. - Requires no dynamic-shape-dependent control-flow. + Requires no dynamic-shape-dependent control-flow.\n - 3: VLLM_COMPILE: Custom vLLM Inductor-based backend with caching, piecewise compilation, shape specialization, and custom passes.""" debug_dump_path: Path | None = None @@ -237,7 +236,7 @@ class CompilationConfig: By default, all custom ops are enabled when running without Inductor and disabled when running with Inductor: mode>=VLLM_COMPILE and use_inductor=True. Inductor generates (fused) Triton kernels for disabled custom ops.""" - splitting_ops: list[str] | None = None + splitting_ops: list[str] = Field(default=None) """A list of ops to exclude from cudagraphs, used in piecewise compilation. The behavior depends on use_inductor_graph_partition: @@ -275,10 +274,8 @@ class CompilationConfig: For future compatibility: If use_inductor is True, backend="inductor" otherwise backend="eager". """ - compile_sizes: list[int | str] | None = None - """Sizes to compile for inductor. In addition - to integers, it also supports "cudagraph_capture_sizes" to - specify the sizes for cudagraph capture.""" + compile_sizes: list[int] = Field(default_factory=list) + """Sizes to compile for inductor.""" inductor_compile_config: dict = field(default_factory=dict) """Additional configurations for inductor. - None: use default configurations.""" @@ -290,7 +287,7 @@ class CompilationConfig: constructor, e.g. `CompilationConfig(inductor_passes={"a": func})`.""" # CudaGraph compilation - cudagraph_mode: CUDAGraphMode | None = None + cudagraph_mode: CUDAGraphMode = Field(default=None) """ The mode of the cudagraph: @@ -495,12 +492,25 @@ def __repr__(self) -> str: __str__ = __repr__ + @field_validator("level", "mode", "splitting_ops", "cudagraph_mode", mode="wrap") + @classmethod + def _skip_none_validation(cls, value: Any, handler: Callable) -> Any: + if value is None: + return value + return handler(value) + + @field_validator("compile_sizes", mode="before") + @classmethod + def _validate_compile_sizes(cls, value: Any) -> Any: + """Deduplicate compile sizes.""" + if isinstance(value, list): + return list(set(value)) + return value + @field_validator("cudagraph_mode", mode="before") @classmethod def validate_cudagraph_mode_before(cls, value: Any) -> Any: - """ - enable parse the `cudagraph_mode` enum type from string - """ + """Enable parsing the `cudagraph_mode` enum type from string.""" if isinstance(value, str): return CUDAGraphMode[value.upper()] return value @@ -692,22 +702,6 @@ def init_with_cudagraph_sizes(self, cudagraph_capture_sizes: list[int]) -> None: ) self.cudagraph_capture_sizes = dedup_sizes - computed_compile_sizes = [] - if self.compile_sizes is not None: - # de-duplicate the sizes provided by the config - self.compile_sizes = list(set(self.compile_sizes)) - for x in self.compile_sizes: - if isinstance(x, str): - assert x == "cudagraph_capture_sizes", ( - "Unrecognized size type in compile_sizes, " - f"expect 'cudagraph_capture_sizes', got {x}" - ) - computed_compile_sizes.extend(self.cudagraph_capture_sizes) - else: - assert isinstance(x, int) - computed_compile_sizes.append(x) - self.compile_sizes = computed_compile_sizes # type: ignore - # sort to make sure cudagraph capture sizes are in descending order self.cudagraph_capture_sizes.sort(reverse=True) self.max_capture_size = ( diff --git a/vllm/config/vllm.py b/vllm/config/vllm.py index fa7310f13b03..f1f961d1293d 100644 --- a/vllm/config/vllm.py +++ b/vllm/config/vllm.py @@ -303,22 +303,10 @@ def __post_init__(self): # we use the default mode. The default mode depends on other # settings (see the below code). if self.compilation_config.mode is None: - if envs.VLLM_USE_V1: - if ( - self.model_config is not None - and not self.model_config.enforce_eager - ): - self.compilation_config.mode = CompilationMode.VLLM_COMPILE - else: - self.compilation_config.mode = CompilationMode.NONE - + if not (self.model_config is None or self.model_config.enforce_eager): + self.compilation_config.mode = CompilationMode.VLLM_COMPILE else: - # NB: Passing both --enforce-eager and a compilation mode - # in V0 means the compilation mode wins out. self.compilation_config.mode = CompilationMode.NONE - else: - assert self.compilation_config.mode >= CompilationMode.NONE - assert self.compilation_config.mode <= CompilationMode.VLLM_COMPILE # If user does not set custom ops via none or all set it here based on # compilation mode and backend. diff --git a/vllm/engine/arg_utils.py b/vllm/engine/arg_utils.py index f6357c5e1ea7..f4da57111a2f 100644 --- a/vllm/engine/arg_utils.py +++ b/vllm/engine/arg_utils.py @@ -19,7 +19,6 @@ TypeAlias, TypeVar, Union, - cast, get_args, get_origin, ) @@ -55,7 +54,6 @@ get_attr_docs, ) from vllm.config.cache import BlockSize, CacheDType, MambaDType, PrefixCachingHashAlgo -from vllm.config.device import Device from vllm.config.model import ( ConvertOption, HfOverrides, @@ -1273,7 +1271,7 @@ def create_engine_config( """ current_platform.pre_register_and_update() - device_config = DeviceConfig(device=cast(Device, current_platform.device_type)) + device_config = DeviceConfig(device=current_platform.device_type) model_config = self.create_model_config() self.model = model_config.model diff --git a/vllm/platforms/cpu.py b/vllm/platforms/cpu.py index 699a56be5cc4..ea403a54a5cf 100644 --- a/vllm/platforms/cpu.py +++ b/vllm/platforms/cpu.py @@ -125,7 +125,7 @@ def get_device_name(cls, device_id: int = 0) -> str: @classmethod def get_attn_backend_cls( cls, - selected_backend: "_Backend", + selected_backend: "_Backend | None", head_size: int, dtype: torch.dtype, kv_cache_dtype: str | None, diff --git a/vllm/platforms/interface.py b/vllm/platforms/interface.py index 098e9058f529..7445440ddfad 100644 --- a/vllm/platforms/interface.py +++ b/vllm/platforms/interface.py @@ -182,7 +182,7 @@ def get_vit_attn_backend(cls, head_size: int, dtype: torch.dtype) -> "_Backend": @classmethod def get_attn_backend_cls( cls, - selected_backend: "_Backend", + selected_backend: "_Backend | None", head_size: int, dtype: torch.dtype, kv_cache_dtype: str | None, diff --git a/vllm/platforms/tpu.py b/vllm/platforms/tpu.py index ab752f438f72..ec3cc045c498 100644 --- a/vllm/platforms/tpu.py +++ b/vllm/platforms/tpu.py @@ -54,7 +54,7 @@ def import_kernels(cls) -> None: @classmethod def get_attn_backend_cls( cls, - selected_backend: "_Backend", + selected_backend: "_Backend | None", head_size: int, dtype: torch.dtype, kv_cache_dtype: str | None, diff --git a/vllm/platforms/xpu.py b/vllm/platforms/xpu.py index cd65cba6b492..ba97c404663a 100644 --- a/vllm/platforms/xpu.py +++ b/vllm/platforms/xpu.py @@ -44,7 +44,7 @@ def import_kernels(cls) -> None: @classmethod def get_attn_backend_cls( cls, - selected_backend: "_Backend", + selected_backend: "_Backend | None", head_size: int, dtype: torch.dtype, kv_cache_dtype: str | None,