Skip to content
Merged
97 changes: 97 additions & 0 deletions tests/v1/kv_offload/tiering/test_fs_tier.py
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,11 @@
import pytest
import torch

from vllm.v1.kv_cache_interface import (
FullAttentionSpec,
KVCacheGroupSpec,
SlidingWindowSpec,
)
from vllm.v1.kv_offload.base import OffloadKey, ReqContext, make_offload_key
from vllm.v1.kv_offload.tiering.base import JobMetadata
from vllm.v1.kv_offload.tiering.fs.manager import (
Expand Down Expand Up @@ -281,3 +286,95 @@ def test_store_load_data_integrity(fs_tier):
assert torch.allclose(tensor[bid], expected[i]), (
f"Block {bid} data mismatch after store+load"
)


# ---------------------------------------------------------------------------
# parallel_agnostic predicate: enabled only for a single full-attention group
# ---------------------------------------------------------------------------


def _mock_spec_with_groups(groups: list, tp: int = 2):
Comment thread
orozery marked this conversation as resolved.
Resolved
vllm_config = MagicMock()
vllm_config.model_config.model = "test-model"
vllm_config.cache_config.block_size = 16
vllm_config.cache_config.cache_dtype = "torch.float32"
vllm_config.parallel_config.tensor_parallel_size = tp
vllm_config.parallel_config.pipeline_parallel_size = 1
vllm_config.parallel_config.prefill_context_parallel_size = 1
vllm_config.parallel_config.decode_context_parallel_size = 1
vllm_config.parallel_config.rank = 0
kv_cache_config = MagicMock()
kv_cache_config.kv_cache_groups = groups
spec = MagicMock()
spec.vllm_config = vllm_config
spec.kv_cache_config = kv_cache_config
spec.block_size_factor = 1
return spec


def _full_attention_group() -> KVCacheGroupSpec:
return KVCacheGroupSpec(
layer_names=["layer0"],
kv_cache_spec=FullAttentionSpec(
block_size=16, num_kv_heads=4, head_size=128, dtype=torch.float32
),
)


def _sliding_window_group() -> KVCacheGroupSpec:
return KVCacheGroupSpec(
layer_names=["layer0"],
kv_cache_spec=SlidingWindowSpec(
block_size=16,
num_kv_heads=4,
head_size=128,
dtype=torch.float32,
sliding_window=128,
),
)


@pytest.fixture
def make_fs_tier(tmp_path):
created: list = []

def _make(groups: list, tp: int = 2) -> FileSystemTierManager:
tensor = _page_aligned_zero_tensor(4, _BLOCK_ELEMENTS)
view = memoryview(tensor.numpy())
root = tmp_path / f"tier{len(created)}"
root.mkdir()
tier = FileSystemTierManager(
offloading_spec=_mock_spec_with_groups(groups, tp),
primary_kv_view=view,
tier_type="fs",
root_dir=str(root),
n_read_threads=2,
n_write_threads=2,
)
created.append((tier, tensor))
return tier

yield _make
for tier, _ in created:
tier.shutdown()


def test_parallel_agnostic_enabled_for_single_full_attention(make_fs_tier):
# A single full-attention group => parallel-agnostic layout: tp/pp/rank are
# collapsed out of the file mapper (tp forced to 1, rank to 0) so the cache
# is shared across tensor-parallel sizes.
tier = make_fs_tier([_full_attention_group()], tp=2)
assert tier.file_mapper.fields["tp_size"] == 1
assert tier.file_mapper.rank == 0


def test_parallel_agnostic_disabled_for_multiple_groups(make_fs_tier):
# More than one KV-cache group (hybrid model) => keep per-layout namespacing.
tier = make_fs_tier([_full_attention_group(), _full_attention_group()], tp=2)
assert tier.file_mapper.fields["tp_size"] == 2


def test_parallel_agnostic_disabled_for_non_full_attention(make_fs_tier):
# Single group but not full attention (sliding window) => keep namespacing.
tier = make_fs_tier([_sliding_window_group()], tp=2)
assert tier.file_mapper.fields["tp_size"] == 2
Comment thread
orozery marked this conversation as resolved.
Resolved
10 changes: 9 additions & 1 deletion vllm/v1/kv_offload/tiering/fs/manager.py
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@
from typing_extensions import override

from vllm.logger import init_logger
from vllm.v1.kv_cache_interface import FullAttentionSpec
from vllm.v1.kv_offload.base import OffloadKey, ReqContext
from vllm.v1.kv_offload.file_mapper import FileMapper
from vllm.v1.kv_offload.tiering.base import (
Expand Down Expand Up @@ -89,11 +90,18 @@ def __init__(
)
self._block_size: int = primary_kv_view.strides[0]

# Create file mapper
# A single full-attention group has a parallelism-invariant offloaded
# block, so share its cache across parallel sizes. Replicated KV (MLA,
# small-GQA) is world_size-scaled and fails closed on the load size check.
kv_cache_groups = offloading_spec.kv_cache_config.kv_cache_groups
parallel_agnostic = len(kv_cache_groups) == 1 and isinstance(
kv_cache_groups[0].kv_cache_spec, FullAttentionSpec
)

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Let's move this logic inside the FileMapper (so other tiers can "enjoy" it as well).
And set parallel_agnostic=True here (and in the obj tier as well)

Copy link
Copy Markdown
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

You're right,
sorry about that.

self.file_mapper = FileMapper.from_offloading_spec(
root_dir=root_dir,
offloading_spec=offloading_spec,
gpu_blocks_per_file=offloading_spec.block_size_factor,
parallel_agnostic=parallel_agnostic,
)

# Write config file
Expand Down
Loading