Skip to content
Merged
22 changes: 21 additions & 1 deletion tests/v1/kv_offload/test_file_mapper.py
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,9 @@

from unittest.mock import MagicMock

import torch

from vllm.v1.kv_cache_interface import KVCacheGroupSpec, MLAAttentionSpec
from vllm.v1.kv_offload.base import (
OffloadingSpec,
make_offload_key,
Expand Down Expand Up @@ -58,7 +61,7 @@ def make_mapper_from_offloading_spec(**kwargs) -> FileMapper:
mock_vllm_config.parallel_config.rank = kwargs.get("rank", 0)

mock_kv_cache_config = MagicMock()
mock_kv_cache_config.kv_cache_groups = []
mock_kv_cache_config.kv_cache_groups = kwargs.get("kv_cache_groups", [])

mock_offloading_spec = MagicMock(spec=OffloadingSpec)
mock_offloading_spec.vllm_config = mock_vllm_config
Expand All @@ -69,6 +72,7 @@ def make_mapper_from_offloading_spec(**kwargs) -> FileMapper:
root_dir=kwargs.get("root_dir", "/tmp/cache"),
offloading_spec=mock_offloading_spec,
gpu_blocks_per_file=mock_offloading_spec.block_size_factor,
parallel_agnostic=kwargs.get("parallel_agnostic", False),
)


Expand Down Expand Up @@ -125,3 +129,19 @@ def test_get_config_file_path():
fm = make_mapper_from_offloading_spec()
config_path = fm.get_config_file_path()
assert config_path == f"{fm.base_path}/config.json"


def test_parallel_agnostic_excludes_mla():
# MLA latent KV is replicated per rank, so its offloaded blocks are not
# parallelism-invariant: the opt-in must not collapse tp/rank.
group = KVCacheGroupSpec(
layer_names=["layer0"],
kv_cache_spec=MLAAttentionSpec(
block_size=16, num_kv_heads=1, head_size=576, dtype=torch.float32
),
)
fm = make_mapper_from_offloading_spec(
tp_size=2, rank=1, kv_cache_groups=[group], parallel_agnostic=True
)
assert fm.fields["tp_size"] == 2
assert fm.rank == 1
97 changes: 97 additions & 0 deletions tests/v1/kv_offload/tiering/test_fs_tier.py
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,11 @@
import pytest
import torch

from vllm.v1.kv_cache_interface import (
FullAttentionSpec,
KVCacheGroupSpec,
SlidingWindowSpec,
)
from vllm.v1.kv_offload.base import OffloadKey, ReqContext, make_offload_key
from vllm.v1.kv_offload.tiering.base import JobMetadata
from vllm.v1.kv_offload.tiering.fs.manager import (
Expand Down Expand Up @@ -281,3 +286,95 @@ def test_store_load_data_integrity(fs_tier):
assert torch.allclose(tensor[bid], expected[i]), (
f"Block {bid} data mismatch after store+load"
)


# ---------------------------------------------------------------------------
# parallel_agnostic predicate: enabled only for a single full-attention group
# ---------------------------------------------------------------------------


def _mock_spec_with_groups(groups: list, tp: int = 2):
Comment thread
orozery marked this conversation as resolved.
Resolved
vllm_config = MagicMock()
vllm_config.model_config.model = "test-model"
vllm_config.cache_config.block_size = 16
vllm_config.cache_config.cache_dtype = "torch.float32"
vllm_config.parallel_config.tensor_parallel_size = tp
vllm_config.parallel_config.pipeline_parallel_size = 1
vllm_config.parallel_config.prefill_context_parallel_size = 1
vllm_config.parallel_config.decode_context_parallel_size = 1
vllm_config.parallel_config.rank = 0
kv_cache_config = MagicMock()
kv_cache_config.kv_cache_groups = groups
spec = MagicMock()
spec.vllm_config = vllm_config
spec.kv_cache_config = kv_cache_config
spec.block_size_factor = 1
return spec


def _full_attention_group() -> KVCacheGroupSpec:
return KVCacheGroupSpec(
layer_names=["layer0"],
kv_cache_spec=FullAttentionSpec(
block_size=16, num_kv_heads=4, head_size=128, dtype=torch.float32
),
)


def _sliding_window_group() -> KVCacheGroupSpec:
return KVCacheGroupSpec(
layer_names=["layer0"],
kv_cache_spec=SlidingWindowSpec(
block_size=16,
num_kv_heads=4,
head_size=128,
dtype=torch.float32,
sliding_window=128,
),
)


@pytest.fixture
def make_fs_tier(tmp_path):
created: list = []

def _make(groups: list, tp: int = 2) -> FileSystemTierManager:
tensor = _page_aligned_zero_tensor(4, _BLOCK_ELEMENTS)
view = memoryview(tensor.numpy())
root = tmp_path / f"tier{len(created)}"
root.mkdir()
tier = FileSystemTierManager(
offloading_spec=_mock_spec_with_groups(groups, tp),
primary_kv_view=view,
tier_type="fs",
root_dir=str(root),
n_read_threads=2,
n_write_threads=2,
)
created.append((tier, tensor))
return tier

yield _make
for tier, _ in created:
tier.shutdown()


def test_parallel_agnostic_enabled_for_single_full_attention(make_fs_tier):
# A single full-attention group => parallel-agnostic layout: tp/pp/rank are
# collapsed out of the file mapper (tp forced to 1, rank to 0) so the cache
# is shared across tensor-parallel sizes.
tier = make_fs_tier([_full_attention_group()], tp=2)
assert tier.file_mapper.fields["tp_size"] == 1
assert tier.file_mapper.rank == 0


def test_parallel_agnostic_disabled_for_multiple_groups(make_fs_tier):
# More than one KV-cache group (hybrid model) => keep per-layout namespacing.
tier = make_fs_tier([_full_attention_group(), _full_attention_group()], tp=2)
assert tier.file_mapper.fields["tp_size"] == 2


def test_parallel_agnostic_disabled_for_non_full_attention(make_fs_tier):
# Single group but not full attention (sliding window) => keep namespacing.
tier = make_fs_tier([_sliding_window_group()], tp=2)
assert tier.file_mapper.fields["tp_size"] == 2
Comment thread
orozery marked this conversation as resolved.
Resolved
10 changes: 10 additions & 0 deletions vllm/v1/kv_offload/file_mapper.py
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@
import hashlib
import json

from vllm.v1.kv_cache_interface import FullAttentionSpec, MLAAttentionSpec
from vllm.v1.kv_offload.base import (
OffloadingSpec,
OffloadKey,
Expand Down Expand Up @@ -81,6 +82,15 @@ def from_offloading_spec(
}
for group in kv_cache_config.kv_cache_groups
]
# Only a single full-attention group is parallelism-invariant. MLA is
# excluded: its latent KV is replicated per rank, never head-sharded.
groups = kv_cache_config.kv_cache_groups
spec = groups[0].kv_cache_spec if len(groups) == 1 else None
parallel_agnostic = (
parallel_agnostic
and isinstance(spec, FullAttentionSpec)
and not isinstance(spec, MLAAttentionSpec)
)
return cls(
root_dir=root_dir,
model_name=vllm_config.model_config.model,
Expand Down
3 changes: 2 additions & 1 deletion vllm/v1/kv_offload/tiering/fs/manager.py
Original file line number Diff line number Diff line change
Expand Up @@ -89,11 +89,12 @@ def __init__(
)
self._block_size: int = primary_kv_view.strides[0]

# Create file mapper
# Opt in; FileMapper enables it only for a parallelism-invariant block.
Comment thread
Etelis marked this conversation as resolved.
Resolved
self.file_mapper = FileMapper.from_offloading_spec(
root_dir=root_dir,
offloading_spec=offloading_spec,
gpu_blocks_per_file=offloading_spec.block_size_factor,
parallel_agnostic=True,
)

# Write config file
Expand Down
Loading