Skip to content
589 changes: 589 additions & 0 deletions docs/design/sleep_mode_tensor_ownership.md

Large diffs are not rendered by default.

282 changes: 279 additions & 3 deletions tests/basic_correctness/test_mem.py
Original file line number Diff line number Diff line change
Expand Up @@ -2,19 +2,124 @@
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project

import asyncio
import os

import pytest
import torch
from torch import nn

import vllm.device_allocator.cumem as cumem
from vllm import LLM, AsyncEngineArgs, AsyncLLMEngine, SamplingParams
from vllm.device_allocator import get_mem_allocator_instance
from vllm.inputs import TokensPrompt
from vllm.platforms import current_platform
from vllm.utils.mem_constants import GiB_bytes

from ..utils import create_new_process_for_each_test, requires_fp8

DEVICE_TYPE = current_platform.device_type

GEMMA3N_SLEEP_WAKE_CYCLES = int(os.getenv("VLLM_SLEEP_WAKE_REPRO_CYCLES", "3"))
GEMMA3N_SLEEP_TENSOR_NAMES = (
"embed_scale",
"embed_scale_per_layer",
"per_layer_input_scale",
"per_layer_projection_scale",
)


def _wake_up_with_poisoned_mappings(allocator, byte_value: int = 0xA5) -> None:
"""Wake discarded allocations with deterministic nonzero contents."""
original_create_and_map = cumem.create_and_map

def create_and_map_with_poison(handle) -> None:
original_create_and_map(handle)
_, size, ptr, _ = handle
cumem.libcudart.cudaMemset(ptr, byte_value, size)

cumem.create_and_map = create_and_map_with_poison
try:
allocator.wake_up()
finally:
cumem.create_and_map = original_create_and_map


def _gemma3n_sleep_tensor_snapshot(model) -> dict[str, tuple[int, float]]:
"""Return addresses and values for Gemma 3n sleep-sensitive tensors."""
from vllm.model_executor.models.gemma3n import Gemma3nSelfDecoder

decoder = next(
(
module
for module in model.modules()
if isinstance(module, Gemma3nSelfDecoder)
),
None,
)
assert decoder is not None
return {
name: (tensor.data_ptr(), tensor.float().item())
for name in GEMMA3N_SLEEP_TENSOR_NAMES
if (tensor := getattr(decoder, name, None)) is not None
}


def _save_gemma3n_dummy_weights(worker) -> None:
model = worker.model_runner.model
worker._gemma3n_dummy_weights = [
(name, parameter.detach().cpu().clone())
for name, parameter in model.named_parameters()
]


def _reload_gemma3n_dummy_weights(worker) -> None:
# GPUModelRunner.reload_weights copies into leaf parameters. Real weight
# loading runs without autograd, so reproduce that context here as well.
with torch.no_grad():
worker.reload_weights(
weights_iterator=iter(worker._gemma3n_dummy_weights),
is_checkpoint_format=False,
)


def _create_tiny_gemma3n_config(model_dir) -> None:
from transformers import Gemma3nTextConfig

config = Gemma3nTextConfig(
architectures=["Gemma3nForCausalLM"],
vocab_size=128,
hidden_size=64,
intermediate_size=[128, 128, 128],
num_hidden_layers=3,
num_attention_heads=4,
num_key_value_heads=2,
head_dim=32,
max_position_embeddings=256,
sliding_window=64,
layer_types=[
"sliding_attention",
"full_attention",
"sliding_attention",
],
vocab_size_per_layer_input=128,
hidden_size_per_layer_input=16,
num_kv_shared_layers=1,
laurel_rank=16,
activation_sparsity_pattern=[0.0, 0.0, 0.0],
)
config.save_pretrained(model_dir)


def _generation_signature(output) -> tuple[tuple[int, ...], tuple[float, ...]]:
completion = output[0].outputs[0]
token_ids = tuple(completion.token_ids)
assert completion.logprobs is not None
chosen_logprobs = tuple(
step[token_id].logprob
for token_id, step in zip(token_ids, completion.logprobs, strict=True)
)
return token_ids, chosen_logprobs


@create_new_process_for_each_test("fork" if current_platform.is_cuda() else "spawn")
def test_python_error():
Expand Down Expand Up @@ -98,10 +203,8 @@ def test_discard_tags():
# Weights are still usable
assert torch.allclose(weights, torch.ones_like(weights))

# Wake up and verify kv_cache is remapped (zeroed content)
# Wake up and verify kv_cache is remapped; discarded contents are undefined.
allocator.wake_up()
# After wake_up the VA is remapped; content is not preserved
# but the allocation is valid
assert kv.shape == (512, 512)

# Full sleep/wake cycle still works after discard
Expand All @@ -110,6 +213,106 @@ def test_discard_tags():
assert torch.allclose(weights, torch.ones_like(weights))


@create_new_process_for_each_test("fork" if current_platform.is_cuda() else "spawn")
@pytest.mark.skipif(current_platform.is_xpu(), reason="Uses the CuMem allocator")
def test_tagged_ordinary_tensor_is_discarded_with_kv_cache():
"""Reproduce allocation-tag contamination with an ordinary torch tensor.

The allocator tags allocations, not semantic tensor owners. This test is a
contract-level reproducer: production code must keep persistent metadata
outside the discardable KV-cache scope.
"""
allocator = get_mem_allocator_instance()

with allocator.use_memory_pool("weights"):
weight = torch.full((4096,), 0x11, dtype=torch.uint8, device=DEVICE_TYPE)
with allocator.use_memory_pool("kv_cache"):
fake_kv = torch.full((4096,), 0x22, dtype=torch.uint8, device=DEVICE_TYPE)
ordinary_tensor = torch.full(
(4096,), 0x33, dtype=torch.uint8, device=DEVICE_TYPE
)

pointers = tuple(t.data_ptr() for t in (weight, fake_kv, ordinary_tensor))
allocator.sleep(offload_tags=("weights",))
_wake_up_with_poisoned_mappings(allocator)
torch.accelerator.synchronize()

assert tuple(t.data_ptr() for t in (weight, fake_kv, ordinary_tensor)) == pointers
assert torch.all(weight == 0x11)
assert torch.all(fake_kv == 0xA5)
assert torch.all(ordinary_tensor == 0xA5)


@create_new_process_for_each_test("fork" if current_platform.is_cuda() else "spawn")
@pytest.mark.skipif(current_platform.is_xpu(), reason="Uses the CuMem allocator")
def test_level2_discards_ordinary_tensor_with_weights_tag():
"""Reproduce the level-2 variant for an ordinary tensor in weights."""
allocator = get_mem_allocator_instance()

with allocator.use_memory_pool("weights"):
fake_weight = torch.full(
(4096,), 0x44, dtype=torch.uint8, device=DEVICE_TYPE
)
ordinary_tensor = torch.full(
(4096,), 0x55, dtype=torch.uint8, device=DEVICE_TYPE
)

pointers = (fake_weight.data_ptr(), ordinary_tensor.data_ptr())
allocator.sleep(offload_tags=())
_wake_up_with_poisoned_mappings(allocator)
torch.accelerator.synchronize()

assert (fake_weight.data_ptr(), ordinary_tensor.data_ptr()) == pointers
assert torch.all(fake_weight == 0xA5)
assert torch.all(ordinary_tensor == 0xA5)


@create_new_process_for_each_test("fork" if current_platform.is_cuda() else "spawn")
@pytest.mark.skipif(current_platform.is_xpu(), reason="Uses CUDA graph and CuMem")
def test_cudagraph_replays_with_corrupted_tagged_constant():
"""Reproduce silent model corruption despite a stable captured pointer."""

class TinyScaleModel(nn.Module):
"""One-operation model with persistent runtime metadata.

``scale`` deliberately remains a plain tensor attribute. Constructing
this model in the KV-cache pool reproduces a persistent model value
accidentally inheriting the cache's discard-on-sleep policy.
"""

def __init__(self) -> None:
super().__init__()
self.scale = torch.tensor([3.0], device=DEVICE_TYPE)

def forward(self, x: torch.Tensor) -> torch.Tensor:
return x * self.scale

allocator = get_mem_allocator_instance()
x = torch.tensor([2.0], device=DEVICE_TYPE)
with allocator.use_memory_pool("kv_cache"):
model = TinyScaleModel()

graph = torch.cuda.CUDAGraph()
with torch.cuda.graph(graph):
output = model(x)
scale_ptr = model.scale.data_ptr()

allocator.sleep(offload_tags=())
_wake_up_with_poisoned_mappings(allocator)
graph.replay()
torch.accelerator.synchronize()

assert model.scale.data_ptr() == scale_ptr
assert not torch.equal(output, torch.tensor([6.0], device=DEVICE_TYPE))

# Owners that cannot move storage out of a discardable scope must recover
# values in place so captured pointers remain valid.
model.scale.fill_(3.0)
graph.replay()
torch.accelerator.synchronize()
assert torch.equal(output, torch.tensor([6.0], device=DEVICE_TYPE))


@create_new_process_for_each_test("fork" if current_platform.is_cuda() else "spawn")
@pytest.mark.skipif(current_platform.is_xpu(), reason="CUDA graph not supported on XPU")
def test_cumem_with_cudagraph():
Expand Down Expand Up @@ -244,6 +447,79 @@ def test_deep_sleep():
assert output[0].outputs[0].text == output2[0].outputs[0].text


@create_new_process_for_each_test()
@pytest.mark.slow_test
@pytest.mark.skipif(
not current_platform.is_cuda(),
reason="Reproduces the CUDA CuMemAllocator level-2 sleep path",
)
def test_gemma3n_level2_sleep_wake_preserves_generation(monkeypatch, tmp_path):
"""Detect naturally occurring Gemma 3n corruption after level-2 sleep."""
monkeypatch.setenv("VLLM_ALLOW_INSECURE_SERIALIZATION", "1")
monkeypatch.setenv("VLLM_ENABLE_V1_MULTIPROCESSING", "0")
_create_tiny_gemma3n_config(tmp_path)

llm = LLM(
model=str(tmp_path),
load_format="dummy",
skip_tokenizer_init=True,
enable_sleep_mode=True,
enforce_eager=True,
attention_backend="TRITON_ATTN",
max_model_len=256,
max_num_seqs=1,
gpu_memory_utilization=0.2,
seed=0,
enable_prefix_caching=False,
disable_log_stats=True,
)
prompt = TokensPrompt(prompt_token_ids=[2, 5, 9, 12, 7, 3])
sampling_params = SamplingParams(temperature=0.0, max_tokens=16, logprobs=1)

before_output = llm.generate(prompt, sampling_params)
before_token_ids, before_logprobs = _generation_signature(before_output)
control_token_ids, control_logprobs = _generation_signature(
llm.generate(prompt, sampling_params)
)
assert control_token_ids == before_token_ids
assert control_logprobs == pytest.approx(before_logprobs, rel=1e-5, abs=1e-5)

before_tensors = llm.apply_model(_gemma3n_sleep_tensor_snapshot)
assert all(
set(snapshot) == set(GEMMA3N_SLEEP_TENSOR_NAMES)
for snapshot in before_tensors
)
llm.collective_rpc(_save_gemma3n_dummy_weights)

for cycle in range(1, GEMMA3N_SLEEP_WAKE_CYCLES + 1):
# Do not poison, zero, or allocate test-side GPU memory between sleep
# and wake. Repeated cycles only increase the chance that the CUDA
# driver naturally returns cleared or reused physical pages.
llm.sleep(level=2)
llm.wake_up(tags=["weights"])
llm.collective_rpc(_reload_gemma3n_dummy_weights)
after_tensors = llm.apply_model(_gemma3n_sleep_tensor_snapshot)

# Check the direct corruption signal before running another forward.
# A corrupted scale can make generate raise inside the model and hide
# the more useful ownership failure behind an unrelated exception.
assert after_tensors == before_tensors, (
f"cycle {cycle}: Gemma 3n runtime tensors changed after sleep/wake"
)

llm.wake_up(tags=["kv_cache"])

after_token_ids, after_logprobs = _generation_signature(
llm.generate(prompt, sampling_params)
)
assert after_token_ids == before_token_ids, (
f"cycle {cycle}: generated tokens changed after sleep/wake"
)
assert after_logprobs == pytest.approx(
before_logprobs, rel=1e-5, abs=1e-5
), f"cycle {cycle}: selected-token logprobs changed after sleep/wake"


@create_new_process_for_each_test()
def test_deep_sleep_lora():
"""Level-2 sleep/wake/reload with enable_lora=True.
Expand Down
36 changes: 36 additions & 0 deletions tests/kernels/core/test_minimax_reduce_rms.py
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,8 @@
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Tests for MiniMax QK RMS-norm: NCCL reference vs Lamport fused kernel."""

from types import SimpleNamespace

import pytest
import torch
import torch.nn as nn
Expand All @@ -20,6 +22,40 @@
from vllm.utils.torch_utils import set_random_seed


def test_lamport_workspace_reset_restores_protocol_state(monkeypatch):
from vllm.model_executor.layers.minimax_rms_norm import lamport_workspace

sentinel_resets = []
monkeypatch.setattr(
lamport_workspace,
"_lamport_fill_neg_zero",
lambda ptr, size: sentinel_resets.append((ptr, size)),
)
owner = object.__new__(lamport_workspace.LamportWorkspace)
owner.world_size = 2
owner.comm_size = 16
owner._lamport = SimpleNamespace(local_ptr=123, serialize=lambda: [11, 22])
owner._flag_buf = torch.tensor([7, 8, 9], dtype=torch.int32)
owner._layout_buf = torch.tensor([99, 99], dtype=torch.int64)
owner._workspace = torch.full((8,), -1, dtype=torch.int64)

owner.reset()

assert sentinel_resets == [(123, 48)]
assert torch.equal(owner._flag_buf, torch.zeros(3, dtype=torch.int32))
assert torch.equal(owner._layout_buf, torch.tensor([0, 16]))
assert owner._workspace.tolist() == [
0,
0,
0,
0,
11,
22,
owner._flag_buf.data_ptr(),
owner._layout_buf.data_ptr(),
]


@ensure_current_vllm_config()
def _worker_forward_qk(
local_rank,
Expand Down
Loading
Loading