Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
30 changes: 30 additions & 0 deletions tests/ut/worker/test_pcp_manager_v2.py
Original file line number Diff line number Diff line change
Expand Up @@ -1039,3 +1039,33 @@ def test_dummy_attention_context_uses_current_batch(pcp_rank, has_stale_batch):
assert manager._global_batch is saved_batch
assert manager._hidden_restore_idx is saved_indices
manager._block_tables.gather_block_tables.assert_not_called()


@pytest.mark.parametrize("slot_dtype", [torch.int32, torch.int64])
def test_pcp_slot_buffers_match_block_tables(slot_dtype):
def init_buffers(manager, **kwargs):
manager._global_batch_slot_mappings = torch.empty((1, 4), dtype=torch.int64)
manager._gathered_kv_slot_mappings = torch.empty((1, 8), dtype=torch.int64)
manager._pad_slot_id = torch.tensor(-1, dtype=torch.int64)
manager.pcp_world_size = 2

block_tables = SimpleNamespace(slot_mappings=torch.empty((1, 4), dtype=slot_dtype))
with patch.object(PCPManager, "__init__", init_buffers):
manager = AscendPCPManager(2, 0, torch.device("cpu"), block_tables=block_tables)

assert manager._global_batch_slot_mappings.dtype == slot_dtype
assert manager._gathered_kv_slot_mappings.dtype == slot_dtype
manager._global_batch_slot_mappings.copy_(torch.tensor([[3200, 3201, -1, 3203]]))
manager._padded_gather_idx = torch.tensor([3, 0, 1, 2])
manager._gathered_kv_write_mask = torch.tensor([True, True, False, True])
buffer_ptr = manager._gathered_kv_slot_mappings.data_ptr()

gathered = manager._convert_to_gathered_slot_mappings(manager._global_batch_slot_mappings)
assert gathered.tolist() == [[3203, 3200, -1, -1]]
assert gathered.dtype == slot_dtype
assert gathered.data_ptr() == buffer_ptr

dummy = manager.get_dummy_slot_mappings(2)
assert dummy.tolist() == [[-1, -1, -1, -1]]
assert dummy.dtype == slot_dtype
assert dummy.data_ptr() == buffer_ptr
14 changes: 14 additions & 0 deletions vllm_ascend/worker/v2/pcp_manager.py
Original file line number Diff line number Diff line change
Expand Up @@ -77,6 +77,20 @@ def __init__(
cp_interleave=cp_interleave,
)

# PCP supplies its own output buffers to compute_slot_mappings, so their
# dtype must match Ascend block-table slots for cache-write operators.
if block_tables is not None:
slot_dtype = block_tables.slot_mappings.dtype
if self._global_batch_slot_mappings is not None:
self._global_batch_slot_mappings = torch.empty_like(
self._global_batch_slot_mappings, dtype=slot_dtype
)
if self._gathered_kv_slot_mappings is not None:
self._gathered_kv_slot_mappings = torch.empty_like(
self._gathered_kv_slot_mappings, dtype=slot_dtype
)
self._pad_slot_id = self._pad_slot_id.to(slot_dtype)
Comment on lines +82 to +92

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

high

To adhere to defensive programming practices and optimize memory usage, we should:

  1. Ensure self._pad_slot_id is not None before calling .to(slot_dtype) to prevent potential runtime errors.
  2. Avoid redundant tensor allocations by checking if the existing dtypes of self._global_batch_slot_mappings and self._gathered_kv_slot_mappings already match slot_dtype before calling torch.empty_like.
Suggested change
if block_tables is not None:
slot_dtype = block_tables.slot_mappings.dtype
if self._global_batch_slot_mappings is not None:
self._global_batch_slot_mappings = torch.empty_like(
self._global_batch_slot_mappings, dtype=slot_dtype
)
if self._gathered_kv_slot_mappings is not None:
self._gathered_kv_slot_mappings = torch.empty_like(
self._gathered_kv_slot_mappings, dtype=slot_dtype
)
self._pad_slot_id = self._pad_slot_id.to(slot_dtype)
if block_tables is not None:
slot_dtype = block_tables.slot_mappings.dtype
if self._global_batch_slot_mappings is not None and self._global_batch_slot_mappings.dtype != slot_dtype:
self._global_batch_slot_mappings = torch.empty_like(
self._global_batch_slot_mappings, dtype=slot_dtype
)
if self._gathered_kv_slot_mappings is not None and self._gathered_kv_slot_mappings.dtype != slot_dtype:
self._gathered_kv_slot_mappings = torch.empty_like(
self._gathered_kv_slot_mappings, dtype=slot_dtype
)
if self._pad_slot_id is not None:
self._pad_slot_id = self._pad_slot_id.to(slot_dtype)


# vLLM #53515 made the PCP-local buffers persistent and uses them for
# graph capture. Preserve that ownership while providing the extra CPU
# and NumPy sequence-length views required by AscendInputBatch.
Expand Down
Loading