From f823fa7a68f269a4c3857626a9431e3165712f9d Mon Sep 17 00:00:00 2001
From: Harry Mellor <19981378+hmellor@users.noreply.github.com>
Date: Thu, 9 Oct 2025 13:48:52 +0200
Subject: [PATCH 01/13] Fix mypy for `vllm/attention` and `vllm/compilation`

This change brings us closer to not needing a custom `mypy` check.

Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
---
 tools/pre_commit/mypy.py               |  4 +--
 vllm/attention/layer.py                |  1 +
 vllm/attention/selector.py             |  9 +++---
 vllm/attention/utils/fa_utils.py       | 27 ++++++++--------
 vllm/compilation/cuda_graph.py         |  1 +
 vllm/compilation/vllm_inductor_pass.py |  2 +-
 vllm/config/compilation.py             | 43 ++++++++++----------------
 vllm/config/device.py                  | 13 ++++++--
 vllm/config/vllm.py                    | 15 ++-------
 vllm/v1/attention/backends/utils.py    |  2 +-
 10 files changed, 52 insertions(+), 65 deletions(-)

diff --git a/tools/pre_commit/mypy.py b/tools/pre_commit/mypy.py
index 22ee08535bdd..fe7b6f13476e 100755
--- a/tools/pre_commit/mypy.py
+++ b/tools/pre_commit/mypy.py
@@ -27,6 +27,8 @@
 FILES = [
     "vllm/*.py",
     "vllm/assets",
+    "vllm/attention",
+    "vllm/compilation",
     "vllm/entrypoints",
     "vllm/inputs",
     "vllm/logging_utils",
@@ -41,8 +43,6 @@
 # from "skip" to "silent", move the following directories to FILES
 SEPARATE_GROUPS = [
     "tests",
-    "vllm/attention",
-    "vllm/compilation",
     "vllm/distributed",
     "vllm/engine",
     "vllm/executor",
diff --git a/vllm/attention/layer.py b/vllm/attention/layer.py
index 9f43cb31218f..81acb903db29 100644
--- a/vllm/attention/layer.py
+++ b/vllm/attention/layer.py
@@ -145,6 +145,7 @@ def __init__(
         `self.kv_cache`.
         """
         super().__init__()
+        sliding_window: Optional[int]
         if per_layer_sliding_window is not None:
             # per-layer sliding window
             sliding_window = per_layer_sliding_window
diff --git a/vllm/attention/selector.py b/vllm/attention/selector.py
index 53677372e055..4971a7295a62 100644
--- a/vllm/attention/selector.py
+++ b/vllm/attention/selector.py
@@ -167,11 +167,8 @@ def _cached_get_attn_backend(
     #
     # THIS SELECTION OVERRIDES THE VLLM_ATTENTION_BACKEND
     # ENVIRONMENT VARIABLE.
-    selected_backend = None
-    backend_by_global_setting: Optional[_Backend] = get_global_forced_attn_backend()
-    if backend_by_global_setting is not None:
-        selected_backend = backend_by_global_setting
-    else:
+    selected_backend = get_global_forced_attn_backend()
+    if selected_backend is None:
         # Check the environment variable and override if specified
         backend_by_env_var: Optional[str] = envs.VLLM_ATTENTION_BACKEND
         if backend_by_env_var is not None:
@@ -190,6 +187,8 @@ def _cached_get_attn_backend(
                     f"Invalid attention backend: '{backend_by_env_var}'. "
                     f"Valid backends are: {list(_Backend.__members__.keys())}"
                 )
+        else:
+            raise ValueError("Attention backend was not forced or set in env var.")
 
     # get device-specific attn_backend
     attention_cls = current_platform.get_attn_backend_cls(
diff --git a/vllm/attention/utils/fa_utils.py b/vllm/attention/utils/fa_utils.py
index e13afd46ee96..8064621e03a1 100644
--- a/vllm/attention/utils/fa_utils.py
+++ b/vllm/attention/utils/fa_utils.py
@@ -9,22 +9,19 @@
 logger = init_logger(__name__)
 
 if current_platform.is_cuda():
-    from vllm import _custom_ops as ops
+    from vllm import _custom_ops
 
-    reshape_and_cache_flash = ops.reshape_and_cache_flash
+    reshape_and_cache_flash = _custom_ops.reshape_and_cache_flash
     from vllm.vllm_flash_attn import flash_attn_varlen_func, get_scheduler_metadata
 elif current_platform.is_xpu():
-    from vllm._ipex_ops import ipex_ops as ops
+    from vllm._ipex_ops import ipex_ops
 
-    reshape_and_cache_flash = ops.reshape_and_cache_flash
-    flash_attn_varlen_func = ops.flash_attn_varlen_func
-    get_scheduler_metadata = ops.get_scheduler_metadata
+    reshape_and_cache_flash = ipex_ops.reshape_and_cache_flash
+    flash_attn_varlen_func = ipex_ops.flash_attn_varlen_func  # type: ignore[assignment]
+    get_scheduler_metadata = ipex_ops.get_scheduler_metadata
 
 
 def get_flash_attn_version(requires_alibi: bool = False) -> Optional[int]:
-    # import here to avoid circular dependencies
-    from vllm.platforms import current_platform
-
     if current_platform.is_xpu():
         return 2
     try:
@@ -75,24 +72,26 @@ def get_flash_attn_version(requires_alibi: bool = False) -> Optional[int]:
 
 
 def flash_attn_supports_fp8() -> bool:
+    device_capability = current_platform.get_device_capability()
     return (
         get_flash_attn_version() == 3
-        and current_platform.get_device_capability().major == 9
+        and device_capability is not None
+        and device_capability.major == 9
     )
 
 
-def flash_attn_supports_mla():
-    from vllm.platforms import current_platform
-
+def flash_attn_supports_mla() -> bool:
     if current_platform.is_cuda():
         try:
             from vllm.vllm_flash_attn.flash_attn_interface import (
                 is_fa_version_supported,
             )
 
+            device_capability = current_platform.get_device_capability()
             return (
                 is_fa_version_supported(3)
-                and current_platform.get_device_capability()[0] == 9
+                and device_capability is not None
+                and device_capability.major == 9
             )
         except (ImportError, AssertionError):
             pass
diff --git a/vllm/compilation/cuda_graph.py b/vllm/compilation/cuda_graph.py
index 4c3ac9e56a37..719bf2bb58e6 100644
--- a/vllm/compilation/cuda_graph.py
+++ b/vllm/compilation/cuda_graph.py
@@ -124,6 +124,7 @@ def __call__(self, *args, **kwargs):
             # runtime modes.
             return self.runnable(*args, **kwargs)
 
+        assert batch_descriptor is not None
         if batch_descriptor not in self.concrete_cudagraph_entries:
             # create a new entry for this batch descriptor
             self.concrete_cudagraph_entries[batch_descriptor] = CUDAGraphEntry(
diff --git a/vllm/compilation/vllm_inductor_pass.py b/vllm/compilation/vllm_inductor_pass.py
index 5aa08220bc2d..06e505a60980 100644
--- a/vllm/compilation/vllm_inductor_pass.py
+++ b/vllm/compilation/vllm_inductor_pass.py
@@ -30,7 +30,7 @@ class VllmInductorPass(InductorPass):
     def __init__(self, config: VllmConfig):
         self.pass_config = config.compilation_config.pass_config
         self.model_dtype = config.model_config.dtype if config.model_config else None
-        self.device = config.device_config.device if config.device_config else None
+        self.device = config.device_config.device
         self.pass_name = self.__class__.__name__
 
     @staticmethod
diff --git a/vllm/config/compilation.py b/vllm/config/compilation.py
index f47fec12d7f9..8d4a40f9f559 100644
--- a/vllm/config/compilation.py
+++ b/vllm/config/compilation.py
@@ -8,7 +8,7 @@
 from pathlib import Path
 from typing import TYPE_CHECKING, Any, Callable, ClassVar, Optional, Union
 
-from pydantic import TypeAdapter, field_validator
+from pydantic import Field, SkipValidation, TypeAdapter, field_validator
 from pydantic.dataclasses import dataclass
 
 from vllm.compilation.inductor_pass import CallableInductorPass, InductorPass
@@ -165,15 +165,15 @@ class CompilationConfig:
     """
 
     # Top-level Compilation control
-    level: Optional[int] = None
+    level: int = Field(default=None, ge=0, le=3)
     """The level of compilation:
 
-    - None: If None, we will select the default compilation level.
-      For V1 engine this is 3, for V0 engine this is 0.
-    - 0: no compilation.
-    - 1: dynamo as is.
-    - 2: dynamo once.
-    - 3: piecewise compilation."""
+    - 0: no compilation.\n
+    - 1: dynamo as is.\n
+    - 2: dynamo once.\n
+    - 3: piecewise compilation.
+
+    If unset, the default compilation level is used, 3."""
     debug_dump_path: Optional[Path] = None
     """The path to dump the debug information."""
     cache_dir: str = ""
@@ -210,7 +210,7 @@ class CompilationConfig:
     By default, all custom ops are enabled when running without Inductor and
     disabled when running with Inductor: level>=PIECEWISE and use_inductor=True.
     Inductor generates (fused) Triton kernels for disabled custom ops."""
-    splitting_ops: Optional[list[str]] = None
+    splitting_ops: list[str] = Field(default=None)
     """A list of ops to split the full graph into subgraphs, used in piecewise
     compilation."""
 
@@ -233,10 +233,9 @@ class CompilationConfig:
     For future compatibility:
     If use_inductor is True, backend="inductor" otherwise backend="eager".
     """
-    compile_sizes: Optional[list[Union[int, str]]] = None
-    """Sizes to compile for inductor. In addition
-    to integers, it also supports "cudagraph_capture_sizes" to
-    specify the sizes for cudagraph capture."""
+    compile_sizes: list[int] = Field(default=None)
+    """Sizes to compile for inductor. If unset, `self.cudagraph_capture_sizes`
+    will be used."""
     inductor_compile_config: dict = field(default_factory=dict)
     """Additional configurations for inductor.
     - None: use default configurations."""
@@ -248,7 +247,7 @@ class CompilationConfig:
     constructor, e.g. `CompilationConfig(inductor_passes={"a": func})`."""
 
     # CudaGraph compilation
-    cudagraph_mode: Optional[CUDAGraphMode] = None
+    cudagraph_mode: SkipValidation[CUDAGraphMode] = None
     """
     The mode of the cudagraph:
 
@@ -616,21 +615,11 @@ def init_with_cudagraph_sizes(self, cudagraph_capture_sizes: list[int]) -> None:
                 )
             self.cudagraph_capture_sizes = dedup_sizes
 
-        computed_compile_sizes = []
-        if self.compile_sizes is not None:
+        if self.compile_sizes is None:
+            self.compile_sizes = self.cudagraph_capture_sizes
+        else:
             # de-duplicate the sizes provided by the config
             self.compile_sizes = list(set(self.compile_sizes))
-            for x in self.compile_sizes:
-                if isinstance(x, str):
-                    assert x == "cudagraph_capture_sizes", (
-                        "Unrecognized size type in compile_sizes, "
-                        f"expect 'cudagraph_capture_sizes', got {x}"
-                    )
-                    computed_compile_sizes.extend(self.cudagraph_capture_sizes)
-                else:
-                    assert isinstance(x, int)
-                    computed_compile_sizes.append(x)
-        self.compile_sizes = computed_compile_sizes  # type: ignore
 
         # sort to make sure cudagraph capture sizes are in descending order
         self.cudagraph_capture_sizes.sort(reverse=True)
diff --git a/vllm/config/device.py b/vllm/config/device.py
index 4b6642479541..f2bc66a856bf 100644
--- a/vllm/config/device.py
+++ b/vllm/config/device.py
@@ -3,10 +3,10 @@
 
 import hashlib
 from dataclasses import field
-from typing import Any, Literal, Optional, Union
+from typing import Any, Literal, Union
 
 import torch
-from pydantic import ConfigDict, SkipValidation
+from pydantic import ConfigDict, field_validator
 from pydantic.dataclasses import dataclass
 
 from vllm.config.utils import config
@@ -19,7 +19,7 @@
 class DeviceConfig:
     """Configuration for the device to use for vLLM execution."""
 
-    device: SkipValidation[Optional[Union[Device, torch.device]]] = "auto"
+    device: Union[str, Device] = "auto"
     """Device type for vLLM execution.
     This parameter is deprecated and will be
     removed in a future release.
@@ -48,6 +48,13 @@ def compute_hash(self) -> str:
         hash_str = hashlib.md5(str(factors).encode(), usedforsecurity=False).hexdigest()
         return hash_str
 
+    @field_validator("device", mode="before")
+    @classmethod
+    def validate_device(cls, v: Any) -> str:
+        if isinstance(v, torch.device):
+            return v.type
+        return v
+
     def __post_init__(self):
         if self.device == "auto":
             # Automated device type detection
diff --git a/vllm/config/vllm.py b/vllm/config/vllm.py
index f6e46bb27013..09bcbe5c172d 100644
--- a/vllm/config/vllm.py
+++ b/vllm/config/vllm.py
@@ -305,23 +305,14 @@ def __post_init__(self):
         # we use the default level. The default level depends on other
         # settings (see the below code).
         if self.compilation_config.level is None:
-            if envs.VLLM_USE_V1:
-                if (
-                    self.model_config is not None
-                    and not self.model_config.enforce_eager
-                ):
-                    self.compilation_config.level = CompilationLevel.PIECEWISE
-                else:
-                    self.compilation_config.level = CompilationLevel.NO_COMPILATION
-
+            if getattr(self.model_config, "enforce_eager", False):
+                self.compilation_config.level = CompilationLevel.PIECEWISE
             else:
-                # NB: Passing both --enforce-eager and a compilation level
-                # in V0 means the compilation level wins out.
                 self.compilation_config.level = CompilationLevel.NO_COMPILATION
+
         else:
             assert self.compilation_config.level >= CompilationLevel.NO_COMPILATION
             assert self.compilation_config.level <= CompilationLevel.PIECEWISE
-            assert self.compilation_config.level <= 3
 
         # If user does not set custom ops via none or all set it here based on
         # compilation level and backend.
diff --git a/vllm/v1/attention/backends/utils.py b/vllm/v1/attention/backends/utils.py
index 638063a8f6f8..488633bfb9f5 100644
--- a/vllm/v1/attention/backends/utils.py
+++ b/vllm/v1/attention/backends/utils.py
@@ -713,7 +713,7 @@ def make_kv_sharing_fast_prefill_common_attn_metadata(
 
 def subclass_attention_backend(
     name_prefix: str,
-    attention_backend_cls: type[AttentionBackend],
+    attention_backend_cls: AttentionBackend,
     builder_cls: type[AttentionMetadataBuilder[M]],
 ) -> type[AttentionBackend]:
     """

From ca030506b83063e2e427e0d385c3f9f8db2aeb53 Mon Sep 17 00:00:00 2001
From: Harry Mellor <19981378+hmellor@users.noreply.github.com>
Date: Thu, 9 Oct 2025 13:51:23 +0200
Subject: [PATCH 02/13] Remove skipvalidation

Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
---
 vllm/config/compilation.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/vllm/config/compilation.py b/vllm/config/compilation.py
index 8d4a40f9f559..0cd154feced9 100644
--- a/vllm/config/compilation.py
+++ b/vllm/config/compilation.py
@@ -8,7 +8,7 @@
 from pathlib import Path
 from typing import TYPE_CHECKING, Any, Callable, ClassVar, Optional, Union
 
-from pydantic import Field, SkipValidation, TypeAdapter, field_validator
+from pydantic import Field, TypeAdapter, field_validator
 from pydantic.dataclasses import dataclass
 
 from vllm.compilation.inductor_pass import CallableInductorPass, InductorPass
@@ -247,7 +247,7 @@ class CompilationConfig:
     constructor, e.g. `CompilationConfig(inductor_passes={"a": func})`."""
 
     # CudaGraph compilation
-    cudagraph_mode: SkipValidation[CUDAGraphMode] = None
+    cudagraph_mode: CUDAGraphMode = Field(default=None)
     """
     The mode of the cudagraph:
 

From b6d07fa0f7880b873dcfcff888f2135644324d91 Mon Sep 17 00:00:00 2001
From: Harry Mellor <19981378+hmellor@users.noreply.github.com>
Date: Thu, 9 Oct 2025 14:08:31 +0200
Subject: [PATCH 03/13] Review comments

Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
---
 vllm/attention/selector.py  | 2 --
 vllm/config/vllm.py         | 2 +-
 vllm/platforms/cpu.py       | 2 +-
 vllm/platforms/interface.py | 2 +-
 vllm/platforms/tpu.py       | 2 +-
 vllm/platforms/xpu.py       | 2 +-
 6 files changed, 5 insertions(+), 7 deletions(-)

diff --git a/vllm/attention/selector.py b/vllm/attention/selector.py
index 4971a7295a62..5264113fab82 100644
--- a/vllm/attention/selector.py
+++ b/vllm/attention/selector.py
@@ -187,8 +187,6 @@ def _cached_get_attn_backend(
                     f"Invalid attention backend: '{backend_by_env_var}'. "
                     f"Valid backends are: {list(_Backend.__members__.keys())}"
                 )
-        else:
-            raise ValueError("Attention backend was not forced or set in env var.")
 
     # get device-specific attn_backend
     attention_cls = current_platform.get_attn_backend_cls(
diff --git a/vllm/config/vllm.py b/vllm/config/vllm.py
index 09bcbe5c172d..7fc286b9040f 100644
--- a/vllm/config/vllm.py
+++ b/vllm/config/vllm.py
@@ -305,7 +305,7 @@ def __post_init__(self):
         # we use the default level. The default level depends on other
         # settings (see the below code).
         if self.compilation_config.level is None:
-            if getattr(self.model_config, "enforce_eager", False):
+            if not (self.model_config is None or self.model_config.enforce_eager):
                 self.compilation_config.level = CompilationLevel.PIECEWISE
             else:
                 self.compilation_config.level = CompilationLevel.NO_COMPILATION
diff --git a/vllm/platforms/cpu.py b/vllm/platforms/cpu.py
index 24e08a8ecbd7..abb99d024515 100644
--- a/vllm/platforms/cpu.py
+++ b/vllm/platforms/cpu.py
@@ -124,7 +124,7 @@ def get_device_name(cls, device_id: int = 0) -> str:
     @classmethod
     def get_attn_backend_cls(
         cls,
-        selected_backend: "_Backend",
+        selected_backend: Optional["_Backend"],
         head_size: int,
         dtype: torch.dtype,
         kv_cache_dtype: Optional[str],
diff --git a/vllm/platforms/interface.py b/vllm/platforms/interface.py
index e372ebf0cb3f..06fa1f9fc804 100644
--- a/vllm/platforms/interface.py
+++ b/vllm/platforms/interface.py
@@ -188,7 +188,7 @@ def get_vit_attn_backend(cls, head_size: int, dtype: torch.dtype) -> "_Backend":
     @classmethod
     def get_attn_backend_cls(
         cls,
-        selected_backend: "_Backend",
+        selected_backend: Optional["_Backend"],
         head_size: int,
         dtype: torch.dtype,
         kv_cache_dtype: Optional[str],
diff --git a/vllm/platforms/tpu.py b/vllm/platforms/tpu.py
index 1c323ba8200a..6b006a1dc6b4 100644
--- a/vllm/platforms/tpu.py
+++ b/vllm/platforms/tpu.py
@@ -54,7 +54,7 @@ def import_kernels(cls) -> None:
     @classmethod
     def get_attn_backend_cls(
         cls,
-        selected_backend: "_Backend",
+        selected_backend: Optional["_Backend"],
         head_size: int,
         dtype: torch.dtype,
         kv_cache_dtype: Optional[str],
diff --git a/vllm/platforms/xpu.py b/vllm/platforms/xpu.py
index e0c8a6605b7d..a6443d3b8bcb 100644
--- a/vllm/platforms/xpu.py
+++ b/vllm/platforms/xpu.py
@@ -44,7 +44,7 @@ def import_kernels(cls) -> None:
     @classmethod
     def get_attn_backend_cls(
         cls,
-        selected_backend: "_Backend",
+        selected_backend: Optional["_Backend"],
         head_size: int,
         dtype: torch.dtype,
         kv_cache_dtype: Optional[str],

From 5e0d99e489590139a255202bdd498b2d757ffd65 Mon Sep 17 00:00:00 2001
From: Harry Mellor <19981378+hmellor@users.noreply.github.com>
Date: Thu, 9 Oct 2025 14:11:17 +0200
Subject: [PATCH 04/13] Review comments 2

Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
---
 vllm/attention/layers/chunked_local_attention.py | 2 +-
 vllm/attention/layers/cross_attention.py         | 2 +-
 vllm/attention/layers/encoder_only_attention.py  | 2 +-
 vllm/v1/attention/backends/utils.py              | 2 +-
 4 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/vllm/attention/layers/chunked_local_attention.py b/vllm/attention/layers/chunked_local_attention.py
index 3d37e901605f..0e2181a49db6 100644
--- a/vllm/attention/layers/chunked_local_attention.py
+++ b/vllm/attention/layers/chunked_local_attention.py
@@ -22,7 +22,7 @@
 
 @functools.lru_cache
 def create_chunked_local_attention_backend(
-    underlying_attn_backend: AttentionBackend,
+    underlying_attn_backend: type[AttentionBackend],
     attention_chunk_size: int,
     block_size: int,
 ) -> type[AttentionBackend]:
diff --git a/vllm/attention/layers/cross_attention.py b/vllm/attention/layers/cross_attention.py
index fb7004f86538..ad384986325c 100644
--- a/vllm/attention/layers/cross_attention.py
+++ b/vllm/attention/layers/cross_attention.py
@@ -81,7 +81,7 @@ def _get_cross_slot_mapping(
 
 @functools.lru_cache
 def create_cross_attention_backend(
-    underlying_attn_backend: AttentionBackend,
+    underlying_attn_backend: type[AttentionBackend],
 ) -> type[AttentionBackend]:
     prefix = "CrossAttention_"
     underlying_builder = underlying_attn_backend.get_builder_cls()
diff --git a/vllm/attention/layers/encoder_only_attention.py b/vllm/attention/layers/encoder_only_attention.py
index f49f195563dc..d79b5d099f7f 100644
--- a/vllm/attention/layers/encoder_only_attention.py
+++ b/vllm/attention/layers/encoder_only_attention.py
@@ -23,7 +23,7 @@
 
 @functools.lru_cache
 def create_encoder_only_attention_backend(
-    underlying_attn_backend: AttentionBackend,
+    underlying_attn_backend: type[AttentionBackend],
 ) -> type[AttentionBackend]:
     prefix = "EncoderOnlyAttention_"
     underlying_builder = underlying_attn_backend.get_builder_cls()
diff --git a/vllm/v1/attention/backends/utils.py b/vllm/v1/attention/backends/utils.py
index 488633bfb9f5..638063a8f6f8 100644
--- a/vllm/v1/attention/backends/utils.py
+++ b/vllm/v1/attention/backends/utils.py
@@ -713,7 +713,7 @@ def make_kv_sharing_fast_prefill_common_attn_metadata(
 
 def subclass_attention_backend(
     name_prefix: str,
-    attention_backend_cls: AttentionBackend,
+    attention_backend_cls: type[AttentionBackend],
     builder_cls: type[AttentionMetadataBuilder[M]],
 ) -> type[AttentionBackend]:
     """

From 27f8ad870fb2e5bafe180920bd3e0955f068b1c4 Mon Sep 17 00:00:00 2001
From: Harry Mellor <19981378+hmellor@users.noreply.github.com>
Date: Thu, 9 Oct 2025 14:24:10 +0200
Subject: [PATCH 05/13] Ignore modules that only exist after install

Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
---
 vllm/attention/layer.py          | 4 +++-
 vllm/attention/utils/fa_utils.py | 5 ++++-
 2 files changed, 7 insertions(+), 2 deletions(-)

diff --git a/vllm/attention/layer.py b/vllm/attention/layer.py
index 81acb903db29..ccb767200927 100644
--- a/vllm/attention/layer.py
+++ b/vllm/attention/layer.py
@@ -104,7 +104,9 @@ def maybe_get_vit_flash_attn_backend(
             if use_upstream_fa:
                 from flash_attn import flash_attn_varlen_func
             else:
-                from vllm.vllm_flash_attn import flash_attn_varlen_func
+                from vllm.vllm_flash_attn import (  # type: ignore[attr-defined]
+                    flash_attn_varlen_func,
+                )
     else:
         flash_attn_varlen_func = None
 
diff --git a/vllm/attention/utils/fa_utils.py b/vllm/attention/utils/fa_utils.py
index 8064621e03a1..cf0e59e4acbe 100644
--- a/vllm/attention/utils/fa_utils.py
+++ b/vllm/attention/utils/fa_utils.py
@@ -12,7 +12,10 @@
     from vllm import _custom_ops
 
     reshape_and_cache_flash = _custom_ops.reshape_and_cache_flash
-    from vllm.vllm_flash_attn import flash_attn_varlen_func, get_scheduler_metadata
+    from vllm.vllm_flash_attn import (  # type: ignore[attr-defined]
+        flash_attn_varlen_func,
+        get_scheduler_metadata,
+    )
 elif current_platform.is_xpu():
     from vllm._ipex_ops import ipex_ops
 

From 1e59be7964225e7156452fb7e0f5082a46a8eebd Mon Sep 17 00:00:00 2001
From: Harry Mellor <19981378+hmellor@users.noreply.github.com>
Date: Thu, 9 Oct 2025 14:27:38 +0200
Subject: [PATCH 06/13] Remove unnecessary whitespace

Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
---
 vllm/config/vllm.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/vllm/config/vllm.py b/vllm/config/vllm.py
index 7fc286b9040f..39a1a95fd019 100644
--- a/vllm/config/vllm.py
+++ b/vllm/config/vllm.py
@@ -309,7 +309,6 @@ def __post_init__(self):
                 self.compilation_config.level = CompilationLevel.PIECEWISE
             else:
                 self.compilation_config.level = CompilationLevel.NO_COMPILATION
-
         else:
             assert self.compilation_config.level >= CompilationLevel.NO_COMPILATION
             assert self.compilation_config.level <= CompilationLevel.PIECEWISE

From 4574efc723d43e44481a29c68cfd26278374cbd9 Mon Sep 17 00:00:00 2001
From: Harry Mellor <19981378+hmellor@users.noreply.github.com>
Date: Thu, 9 Oct 2025 14:40:43 +0200
Subject: [PATCH 07/13] Remove now redundant cast

Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
---
 vllm/engine/arg_utils.py | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/vllm/engine/arg_utils.py b/vllm/engine/arg_utils.py
index 7e66d8dba8ac..72c86f5b3004 100644
--- a/vllm/engine/arg_utils.py
+++ b/vllm/engine/arg_utils.py
@@ -18,7 +18,6 @@
     Optional,
     TypeVar,
     Union,
-    cast,
     get_args,
     get_origin,
 )
@@ -53,7 +52,6 @@
     get_attr_docs,
 )
 from vllm.config.cache import BlockSize, CacheDType, MambaDType, PrefixCachingHashAlgo
-from vllm.config.device import Device
 from vllm.config.model import (
     ConvertOption,
     HfOverrides,
@@ -1234,7 +1232,7 @@ def create_engine_config(
         """
         current_platform.pre_register_and_update()
 
-        device_config = DeviceConfig(device=cast(Device, current_platform.device_type))
+        device_config = DeviceConfig(device=current_platform.device_type)
 
         model_config = self.create_model_config()
         self.model = model_config.model

From 1170aa910783226ac3fe7cc04f47299ec9d4089e Mon Sep 17 00:00:00 2001
From: Harry Mellor <19981378+hmellor@users.noreply.github.com>
Date: Fri, 17 Oct 2025 00:41:09 +0200
Subject: [PATCH 08/13] Fix mypy errors

Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
---
 vllm/compilation/collective_fusion.py    | 14 +++++++-------
 vllm/compilation/sequence_parallelism.py | 20 ++++++++++++++++----
 vllm/config/compilation.py               | 17 ++++++++---------
 vllm/config/device.py                    |  9 +--------
 vllm/platforms/interface.py              |  2 +-
 5 files changed, 33 insertions(+), 29 deletions(-)

diff --git a/vllm/compilation/collective_fusion.py b/vllm/compilation/collective_fusion.py
index 7c85c89bcd7a..824b5b7ac4fa 100644
--- a/vllm/compilation/collective_fusion.py
+++ b/vllm/compilation/collective_fusion.py
@@ -49,7 +49,7 @@
 
 
 class BasePattern:
-    def __init__(self, dtype: torch.dtype, device: str):
+    def __init__(self, dtype: torch.dtype, device: torch.Device):
         self.dtype = dtype
         self.device = device
         self.tp = get_tp_group()
@@ -663,7 +663,7 @@ def __init__(
         self,
         epsilon: float,
         dtype: torch.dtype,
-        device: str,
+        device: torch.Device,
         allreduce_params: FlashInferFusedAllReduceParams,
     ):
         super().__init__(dtype, device)
@@ -727,7 +727,7 @@ def __init__(
         self,
         epsilon: float,
         dtype: torch.dtype,
-        device: str,
+        device: torch.Device,
         allreduce_params: FlashInferFusedAllReduceParams,
     ):
         super().__init__(dtype, device)
@@ -792,7 +792,7 @@ def __init__(
         self,
         epsilon: float,
         dtype: torch.dtype,
-        device: str,
+        device: torch.Device,
         allreduce_params: FlashInferFusedAllReduceParams,
     ):
         super().__init__(dtype, device)
@@ -884,7 +884,7 @@ def __init__(
         self,
         epsilon: float,
         dtype: torch.dtype,
-        device: str,
+        device: torch.Device,
         allreduce_params: FlashInferFusedAllReduceParams,
     ):
         super().__init__(dtype, device)
@@ -980,7 +980,7 @@ def __init__(
         self,
         epsilon: float,
         dtype: torch.dtype,
-        device: str,
+        device: torch.Device,
         allreduce_params: FlashInferFusedAllReduceParams,
     ):
         super().__init__(dtype, device)
@@ -1084,7 +1084,7 @@ def __init__(
         self,
         epsilon: float,
         dtype: torch.dtype,
-        device: str,
+        device: torch.Device,
         allreduce_params: FlashInferFusedAllReduceParams,
     ):
         super().__init__(dtype, device)
diff --git a/vllm/compilation/sequence_parallelism.py b/vllm/compilation/sequence_parallelism.py
index 31624a8fdcc0..fb6fab9808cf 100644
--- a/vllm/compilation/sequence_parallelism.py
+++ b/vllm/compilation/sequence_parallelism.py
@@ -108,7 +108,7 @@ def __init__(
         self,
         epsilon: float,
         dtype: torch.dtype,
-        device: str,
+        device: torch.Device,
         quant_op: torch._ops.OpOverload | None = None,
         **kwargs,
     ):
@@ -257,7 +257,11 @@ def replacement(
 
 class FirstAllReduceRMSNormStaticFP8Pattern(_SequenceParallelPatternHelper):
     def __init__(
-        self, epsilon: float, dtype: torch.dtype, device: str, op: torch._ops.OpOverload
+        self,
+        epsilon: float,
+        dtype: torch.dtype,
+        device: torch.Device,
+        op: torch._ops.OpOverload,
     ):
         super().__init__(epsilon, dtype, device, quant_op=op)
 
@@ -313,7 +317,11 @@ def replacement(
 
 class MiddleAllReduceRMSNormStaticFP8Pattern(_SequenceParallelPatternHelper):
     def __init__(
-        self, epsilon: float, dtype: torch.dtype, device: str, op: torch._ops.OpOverload
+        self,
+        epsilon: float,
+        dtype: torch.dtype,
+        device: torch.Device,
+        op: torch._ops.OpOverload,
     ):
         super().__init__(epsilon, dtype, device, quant_op=op)
 
@@ -373,7 +381,11 @@ def replacement(
 
 class LastAllReduceRMSNormStaticFP8Pattern(_SequenceParallelPatternHelper):
     def __init__(
-        self, epsilon: float, dtype: torch.dtype, device: str, op: torch._ops.OpOverload
+        self,
+        epsilon: float,
+        dtype: torch.dtype,
+        device: torch.Device,
+        op: torch._ops.OpOverload,
     ):
         super().__init__(epsilon, dtype, device, quant_op=op)
 
diff --git a/vllm/config/compilation.py b/vllm/config/compilation.py
index 0fea0a851211..706e58b915c3 100644
--- a/vllm/config/compilation.py
+++ b/vllm/config/compilation.py
@@ -178,19 +178,18 @@ class CompilationConfig:
     """
 
     # Top-level Compilation control
-    level: int | None = Field(default=None, ge=0, le=3)
+    level: int = Field(default=None, ge=0, le=3)
     """
     Level is deprecated and will be removed in the next release,
     either 0.12.0 or 0.11.2 whichever is soonest.
     Please use mode. Currently all levels are mapped to mode.
     """
     # Top-level Compilation control
-    mode: int | None = Field(default=None, ge=0, le=3)
-    """The compilation approach used for torch.compile-based compilation of the
-    model.
+    mode: int = Field(default=None, ge=0, le=3)
+    """
+    The compilation approach used for torch.compile-based compilation of the model.
+    If unset, we will select the default compilation mode. For V1 engine this is 3.
 
-    - None: If None, we will select the default compilation mode.
-      For V1 engine this is 3.\n
     - 0: NONE: No torch.compile compilation is applied, model runs in fully
          eager pytorch mode. The model runs as-is.\n
     - 1: STOCK_TORCH_COMPILE: The standard `torch.compile` compilation pipeline.\n
@@ -236,7 +235,7 @@ class CompilationConfig:
     By default, all custom ops are enabled when running without Inductor and
     disabled when running with Inductor: mode>=VLLM_COMPILE and use_inductor=True.
     Inductor generates (fused) Triton kernels for disabled custom ops."""
-    splitting_ops: list[str] | None = None
+    splitting_ops: list[str] = Field(default=None)
     """A list of ops to exclude from cudagraphs, used in piecewise compilation.
 
     The behavior depends on use_inductor_graph_partition:
@@ -274,7 +273,7 @@ class CompilationConfig:
     For future compatibility:
     If use_inductor is True, backend="inductor" otherwise backend="eager".
     """
-    compile_sizes: list[int] | None = None
+    compile_sizes: list[int] = Field(default=None)
     """Sizes to compile for inductor. If unset, `self.cudagraph_capture_sizes`
     will be used."""
     inductor_compile_config: dict = field(default_factory=dict)
@@ -288,7 +287,7 @@ class CompilationConfig:
     constructor, e.g. `CompilationConfig(inductor_passes={"a": func})`."""
 
     # CudaGraph compilation
-    cudagraph_mode: CUDAGraphMode | None = None
+    cudagraph_mode: CUDAGraphMode = Field(default=None)
     """
     The mode of the cudagraph:
 
diff --git a/vllm/config/device.py b/vllm/config/device.py
index e9ee52320e53..e85cd15de8cf 100644
--- a/vllm/config/device.py
+++ b/vllm/config/device.py
@@ -6,7 +6,7 @@
 from typing import Any, Literal
 
 import torch
-from pydantic import ConfigDict, SkipValidation, field_validator
+from pydantic import ConfigDict, SkipValidation
 from pydantic.dataclasses import dataclass
 
 from vllm.config.utils import config
@@ -48,13 +48,6 @@ def compute_hash(self) -> str:
         hash_str = hashlib.md5(str(factors).encode(), usedforsecurity=False).hexdigest()
         return hash_str
 
-    @field_validator("device", mode="before")
-    @classmethod
-    def validate_device(cls, v: Any) -> str:
-        if isinstance(v, torch.device):
-            return v.type
-        return v
-
     def __post_init__(self):
         if self.device == "auto":
             # Automated device type detection
diff --git a/vllm/platforms/interface.py b/vllm/platforms/interface.py
index f9f2cc4d34e2..bb12c64c82ba 100644
--- a/vllm/platforms/interface.py
+++ b/vllm/platforms/interface.py
@@ -186,7 +186,7 @@ def get_vit_attn_backend(cls, head_size: int, dtype: torch.dtype) -> _Backend:
     @classmethod
     def get_attn_backend_cls(
         cls,
-        selected_backend: _Backend,
+        selected_backend: _Backend | None,
         head_size: int,
         dtype: torch.dtype,
         kv_cache_dtype: str | None,

From 80bb357a446a92c702bd0b204587e1eddac39b3b Mon Sep 17 00:00:00 2001
From: Harry Mellor <19981378+hmellor@users.noreply.github.com>
Date: Fri, 17 Oct 2025 12:42:02 +0200
Subject: [PATCH 09/13] `torch.Device` -> `torch.types.Device`

Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
---
 vllm/compilation/collective_fusion.py    | 14 +++++++-------
 vllm/compilation/sequence_parallelism.py |  8 ++++----
 2 files changed, 11 insertions(+), 11 deletions(-)

diff --git a/vllm/compilation/collective_fusion.py b/vllm/compilation/collective_fusion.py
index 824b5b7ac4fa..e142dbacd630 100644
--- a/vllm/compilation/collective_fusion.py
+++ b/vllm/compilation/collective_fusion.py
@@ -49,7 +49,7 @@
 
 
 class BasePattern:
-    def __init__(self, dtype: torch.dtype, device: torch.Device):
+    def __init__(self, dtype: torch.dtype, device: torch.types.Device):
         self.dtype = dtype
         self.device = device
         self.tp = get_tp_group()
@@ -663,7 +663,7 @@ def __init__(
         self,
         epsilon: float,
         dtype: torch.dtype,
-        device: torch.Device,
+        device: torch.types.Device,
         allreduce_params: FlashInferFusedAllReduceParams,
     ):
         super().__init__(dtype, device)
@@ -727,7 +727,7 @@ def __init__(
         self,
         epsilon: float,
         dtype: torch.dtype,
-        device: torch.Device,
+        device: torch.types.Device,
         allreduce_params: FlashInferFusedAllReduceParams,
     ):
         super().__init__(dtype, device)
@@ -792,7 +792,7 @@ def __init__(
         self,
         epsilon: float,
         dtype: torch.dtype,
-        device: torch.Device,
+        device: torch.types.Device,
         allreduce_params: FlashInferFusedAllReduceParams,
     ):
         super().__init__(dtype, device)
@@ -884,7 +884,7 @@ def __init__(
         self,
         epsilon: float,
         dtype: torch.dtype,
-        device: torch.Device,
+        device: torch.types.Device,
         allreduce_params: FlashInferFusedAllReduceParams,
     ):
         super().__init__(dtype, device)
@@ -980,7 +980,7 @@ def __init__(
         self,
         epsilon: float,
         dtype: torch.dtype,
-        device: torch.Device,
+        device: torch.types.Device,
         allreduce_params: FlashInferFusedAllReduceParams,
     ):
         super().__init__(dtype, device)
@@ -1084,7 +1084,7 @@ def __init__(
         self,
         epsilon: float,
         dtype: torch.dtype,
-        device: torch.Device,
+        device: torch.types.Device,
         allreduce_params: FlashInferFusedAllReduceParams,
     ):
         super().__init__(dtype, device)
diff --git a/vllm/compilation/sequence_parallelism.py b/vllm/compilation/sequence_parallelism.py
index fb6fab9808cf..c4a4b53f87e7 100644
--- a/vllm/compilation/sequence_parallelism.py
+++ b/vllm/compilation/sequence_parallelism.py
@@ -108,7 +108,7 @@ def __init__(
         self,
         epsilon: float,
         dtype: torch.dtype,
-        device: torch.Device,
+        device: torch.types.Device,
         quant_op: torch._ops.OpOverload | None = None,
         **kwargs,
     ):
@@ -260,7 +260,7 @@ def __init__(
         self,
         epsilon: float,
         dtype: torch.dtype,
-        device: torch.Device,
+        device: torch.types.Device,
         op: torch._ops.OpOverload,
     ):
         super().__init__(epsilon, dtype, device, quant_op=op)
@@ -320,7 +320,7 @@ def __init__(
         self,
         epsilon: float,
         dtype: torch.dtype,
-        device: torch.Device,
+        device: torch.types.Device,
         op: torch._ops.OpOverload,
     ):
         super().__init__(epsilon, dtype, device, quant_op=op)
@@ -384,7 +384,7 @@ def __init__(
         self,
         epsilon: float,
         dtype: torch.dtype,
-        device: torch.Device,
+        device: torch.types.Device,
         op: torch._ops.OpOverload,
     ):
         super().__init__(epsilon, dtype, device, quant_op=op)

From b456ee8fb3e68d9429f21d50651ff11c52b88080 Mon Sep 17 00:00:00 2001
From: Harry Mellor <19981378+hmellor@users.noreply.github.com>
Date: Fri, 17 Oct 2025 16:19:05 +0200
Subject: [PATCH 10/13] Review comment aboud compile_sizes

Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
---
 vllm/config/compilation.py | 17 ++++++++---------
 1 file changed, 8 insertions(+), 9 deletions(-)

diff --git a/vllm/config/compilation.py b/vllm/config/compilation.py
index 276238242826..0ccc3d3bb3df 100644
--- a/vllm/config/compilation.py
+++ b/vllm/config/compilation.py
@@ -274,9 +274,8 @@ class CompilationConfig:
     For future compatibility:
     If use_inductor is True, backend="inductor" otherwise backend="eager".
     """
-    compile_sizes: list[int] = Field(default=None)
-    """Sizes to compile for inductor. If unset, `self.cudagraph_capture_sizes`
-    will be used."""
+    compile_sizes: list[int] = Field(default_factory=list)
+    """Sizes to compile for inductor."""
     inductor_compile_config: dict = field(default_factory=dict)
     """Additional configurations for inductor.
     - None: use default configurations."""
@@ -682,12 +681,6 @@ def init_with_cudagraph_sizes(self, cudagraph_capture_sizes: list[int]) -> None:
                 )
             self.cudagraph_capture_sizes = dedup_sizes
 
-        if self.compile_sizes is None:
-            self.compile_sizes = self.cudagraph_capture_sizes
-        else:
-            # de-duplicate the sizes provided by the config
-            self.compile_sizes = list(set(self.compile_sizes))
-
         # sort to make sure cudagraph capture sizes are in descending order
         self.cudagraph_capture_sizes.sort(reverse=True)
         self.max_capture_size = (
@@ -706,6 +699,12 @@ def init_with_cudagraph_sizes(self, cudagraph_capture_sizes: list[int]) -> None:
                     self.bs_to_padded_graph_size[bs] = end
         self.bs_to_padded_graph_size[self.max_capture_size] = self.max_capture_size
 
+    @field_validator("compile_sizes")
+    @classmethod
+    def _validate_compile_sizes(cls, value: list[int]) -> list[int]:
+        """Deduplicate compile sizes."""
+        return list(set(value))
+
     def set_splitting_ops_for_v1(self):
         # NOTE: this function needs to be called only when mode is
         # CompilationMode.VLLM_COMPILE

From 60640a8727fbc27ec12592487b2e37a6d6ab4bb2 Mon Sep 17 00:00:00 2001
From: Harry Mellor <19981378+hmellor@users.noreply.github.com>
Date: Fri, 17 Oct 2025 16:24:50 +0200
Subject: [PATCH 11/13] Make it a before validator

Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
---
 vllm/config/compilation.py | 18 +++++++++---------
 1 file changed, 9 insertions(+), 9 deletions(-)

diff --git a/vllm/config/compilation.py b/vllm/config/compilation.py
index 0ccc3d3bb3df..49672b8c53ab 100644
--- a/vllm/config/compilation.py
+++ b/vllm/config/compilation.py
@@ -484,12 +484,18 @@ def __repr__(self) -> str:
 
     __str__ = __repr__
 
+    @field_validator("compile_sizes", mode="before")
+    @classmethod
+    def _validate_compile_sizes(cls, value: Any) -> Any:
+        """Deduplicate compile sizes."""
+        if isinstance(value, list):
+            return list(set(value))
+        return value
+
     @field_validator("cudagraph_mode", mode="before")
     @classmethod
     def validate_cudagraph_mode_before(cls, value: Any) -> Any:
-        """
-        enable parse the `cudagraph_mode` enum type from string
-        """
+        """Enable parsing the `cudagraph_mode` enum type from string."""
         if isinstance(value, str):
             return CUDAGraphMode[value.upper()]
         return value
@@ -699,12 +705,6 @@ def init_with_cudagraph_sizes(self, cudagraph_capture_sizes: list[int]) -> None:
                     self.bs_to_padded_graph_size[bs] = end
         self.bs_to_padded_graph_size[self.max_capture_size] = self.max_capture_size
 
-    @field_validator("compile_sizes")
-    @classmethod
-    def _validate_compile_sizes(cls, value: list[int]) -> list[int]:
-        """Deduplicate compile sizes."""
-        return list(set(value))
-
     def set_splitting_ops_for_v1(self):
         # NOTE: this function needs to be called only when mode is
         # CompilationMode.VLLM_COMPILE

From e19b02ccc49df6049196ae276ee571b2ca88890c Mon Sep 17 00:00:00 2001
From: Harry Mellor <19981378+hmellor@users.noreply.github.com>
Date: Tue, 21 Oct 2025 11:17:13 +0200
Subject: [PATCH 12/13] Skip `None` validation for delayed defaults

Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
---
 vllm/config/compilation.py | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/vllm/config/compilation.py b/vllm/config/compilation.py
index 1493ce07f5d8..480aebe58678 100644
--- a/vllm/config/compilation.py
+++ b/vllm/config/compilation.py
@@ -492,6 +492,13 @@ def __repr__(self) -> str:
 
     __str__ = __repr__
 
+    @field_validator("level", "mode", "splitting_ops", "cudagraph_mode", mode="wrap")
+    @classmethod
+    def _skip_none_validation(cls, value: Any, handler: Callable) -> Any:
+        if value is None:
+            return value
+        return handler(value)
+
     @field_validator("compile_sizes", mode="before")
     @classmethod
     def _validate_compile_sizes(cls, value: Any) -> Any:

From 65b3d74e38cb1f5f8fa5df42e63abfba56426bc0 Mon Sep 17 00:00:00 2001
From: Harry Mellor <19981378+hmellor@users.noreply.github.com>
Date: Tue, 21 Oct 2025 12:30:04 +0200
Subject: [PATCH 13/13] Fix type hint for get_kv_cache_spec

Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
---
 vllm/attention/layer.py                         | 2 +-
 vllm/attention/layers/encoder_only_attention.py | 3 +--
 2 files changed, 2 insertions(+), 3 deletions(-)

diff --git a/vllm/attention/layer.py b/vllm/attention/layer.py
index d5fa6c71501d..b331abc3c365 100644
--- a/vllm/attention/layer.py
+++ b/vllm/attention/layer.py
@@ -410,7 +410,7 @@ def process_weights_after_loading(self, act_dtype: torch.dtype):
     def get_attn_backend(self) -> type[AttentionBackend]:
         return self.attn_backend
 
-    def get_kv_cache_spec(self, vllm_config: VllmConfig) -> KVCacheSpec:
+    def get_kv_cache_spec(self, vllm_config: VllmConfig) -> KVCacheSpec | None:
         # Block size may get updated after model loading, refresh it
         block_size = vllm_config.cache_config.block_size
         # Should not be called for enc-dec or encoder-only attention.
diff --git a/vllm/attention/layers/encoder_only_attention.py b/vllm/attention/layers/encoder_only_attention.py
index e83853abd6c3..0ac838df2854 100644
--- a/vllm/attention/layers/encoder_only_attention.py
+++ b/vllm/attention/layers/encoder_only_attention.py
@@ -21,7 +21,6 @@
     CommonAttentionMetadata,
     subclass_attention_backend,
 )
-from vllm.v1.kv_cache_interface import KVCacheSpec
 
 
 @functools.lru_cache
@@ -103,6 +102,6 @@ def __init__(
             **kwargs,
         )
 
-    def get_kv_cache_spec(self, vllm_config: VllmConfig) -> KVCacheSpec:
+    def get_kv_cache_spec(self, vllm_config: VllmConfig) -> None:
         # Does not need KV cache
         return None