Skip to content
Merged
61 changes: 61 additions & 0 deletions tests/v1/worker/test_attn_utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -9,11 +9,13 @@

from types import SimpleNamespace

import numpy as np
import pytest
import torch

import vllm.v1.hisparse.binding as attn_utils_module
from tests.v1.attention.utils import dense_kv_cache_views
from vllm.config.compilation import CUDAGraphMode
from vllm.v1.attention.backend import AttentionBackend, AttentionCGSupport, MultipleOf
from vllm.v1.core.kv_cache_utils import KVCacheBlockCopy
from vllm.v1.hisparse.binding import allocate_hisparse_kv_caches
Expand All @@ -30,9 +32,11 @@
)
from vllm.v1.worker.gpu import attn_utils
from vllm.v1.worker.gpu.attn_utils import (
FastPrefillHelper,
get_attn_cg_support,
get_query_lens_mismatch_unsupported_backend,
)
from vllm.v1.worker.gpu.cudagraph_utils import BatchExecutionDescriptor
from vllm.v1.worker.utils import (
AttentionGroup,
allocate_kv_cache,
Expand Down Expand Up @@ -240,6 +244,63 @@ def check(
assert attn_utils.get_kv_sharing_fast_prefill_eligible_layers(vllm_config) == set()


@pytest.mark.parametrize(
"num_tokens", [1, 2, 3, 4, 7, 8, 15, 16, 31, 32, 63, 64, 127, 128]
)
@pytest.mark.parametrize("num_active_loras", [1, 2, 4])
def test_fast_prefill_dispatch_preserves_active_lora_count(
num_tokens: int, num_active_loras: int
):
"""Fast-prefill padding must match the main dispatch's LoRA variant."""

class FakeCudaGraphManager:
device = "cpu"

def __init__(self):
self.dispatch_calls = []

def dispatch(self, **kwargs):
self.dispatch_calls.append(kwargs)
# A captured no-LoRA graph is padded to 8 tokens, while the
# active-LoRA path stays eager at the unpadded token count.
if kwargs["num_active_loras"] == 0:
num_tokens = 8
mode = CUDAGraphMode.PIECEWISE
else:
num_tokens = kwargs["num_tokens"]
mode = CUDAGraphMode.NONE
return BatchExecutionDescriptor(
cg_mode=mode,
num_tokens=num_tokens,
num_reqs=kwargs["num_reqs"],
num_active_loras=kwargs["num_active_loras"],
num_ubatches=1,
)

manager = FakeCudaGraphManager()
helper = FastPrefillHelper(manager, max_num_tokens=max(32, num_tokens))
metadata = helper.prepare(
torch.arange(num_tokens, dtype=torch.int32),
num_reqs=1,
cu_num_logits_np=np.array([0, num_tokens], dtype=np.int32),
has_prefill=True,
batch_desc=BatchExecutionDescriptor(
cg_mode=CUDAGraphMode.NONE,
num_tokens=num_tokens,
num_reqs=1,
num_active_loras=num_active_loras,
num_ubatches=1,
),
num_active_loras=num_active_loras,
)

assert metadata is not None
assert metadata.num_logits_indices == num_tokens
assert metadata.logits_indices_padded.shape[0] == num_tokens
assert metadata.max_logits_per_req == num_tokens
assert manager.dispatch_calls[-1]["num_active_loras"] == num_active_loras


class _FakeSharedHostRegion:
def __init__(self) -> None:
self.cleanup_calls = 0
Expand Down
3 changes: 2 additions & 1 deletion vllm/v1/worker/gpu/attn_utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -92,6 +92,7 @@ def prepare(
cu_num_logits_np: np.ndarray,
has_prefill: bool,
batch_desc: "BatchExecutionDescriptor",
num_active_loras: int = 0,
) -> FastPrefillBatchMetadata | None:
if (
not has_prefill
Expand All @@ -113,7 +114,7 @@ def prepare(
num_reqs=num_reqs,
num_tokens=num_logits,
uniform_token_count=None,
num_active_loras=0,
num_active_loras=num_active_loras,
)
num_logits_padded = min(desc.num_tokens, self.max_num_tokens)
return FastPrefillBatchMetadata(
Expand Down
7 changes: 6 additions & 1 deletion vllm/v1/worker/gpu/model_runner.py
Original file line number Diff line number Diff line change
Expand Up @@ -1258,6 +1258,7 @@ def prepare_inputs(
scheduler_output: SchedulerOutput,
batch_req_state: "BatchReqState",
batch_desc: BatchExecutionDescriptor,
num_active_loras: int,
) -> InputBatch:
num_tokens = batch_req_state.num_tokens
num_tokens_after_padding = max(num_tokens, batch_desc.num_tokens)
Expand Down Expand Up @@ -1393,6 +1394,7 @@ def prepare_inputs(
cu_num_logits_np,
batch_req_state.has_prefill,
batch_desc,
num_active_loras=num_active_loras,
)

# CPU upper bound on seq_lens; padded entries left at zero.
Expand Down Expand Up @@ -1697,7 +1699,10 @@ def execute_model(
cudagraph_stats = make_cudagraph_stats(batch_desc, num_toks)
assert batch_req_state is not None
input_batch = self.prepare_inputs(
scheduler_output, batch_req_state, batch_desc
scheduler_output,
batch_req_state,
batch_desc,
num_active_loras,
)
block_tables, slot_mappings = self.prepare_attn(input_batch)
# Mamba "align" pre-copy: migrate recurrent state across block
Expand Down
7 changes: 6 additions & 1 deletion vllm/v1/worker/mm_encoder_model_runner.py
Original file line number Diff line number Diff line change
Expand Up @@ -117,7 +117,12 @@ def execute_model(
num_tokens=batch_req_state.num_tokens,
num_reqs=None,
)
self.prepare_inputs(scheduler_output, batch_req_state, batch_desc)
self.prepare_inputs(
scheduler_output,
batch_req_state,
batch_desc,
num_active_loras=0,
)

scheduled_encoder_inputs = scheduler_output.scheduled_encoder_inputs
if self.lora_config is not None:
Expand Down
Loading