Repository navigation
BugFix(PCP): use persistent input buffers for PIECEWISE CUDA graphs #53515
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Merged
LucasWilkinson
merged 10 commits into
vllm-project:main
from
pisceskkk:codex/pcp-piecewise-debug-main
Aug 26, 2026
Merged
Changes from 6 commits
Commits
Show all changes
10 commits
Select commit
Hold shift + click to select a range
c82aa7d
fix(pcp): stabilize piecewise cudagraph replay
pisceskkk ce62292
refactor(pcp): minimize piecewise graph fix
pisceskkk a8aa573
fix(pcp): preserve cudagraph token padding
pisceskkk 48ac3cc
fix(pcp): reuse capture-bound padding mask
pisceskkk 3cfe87e
fix(pcp): capture graphs with PCP input buffers
pisceskkk 1dfcbc2
perf(pcp): dispatch graphs by rank-local token count
pisceskkk 8e5fa50
Update vllm/v1/worker/gpu/model_runner.py
LucasWilkinson 8bdd185
Update vllm/v1/worker/gpu/model_runner.py
LucasWilkinson 31c016c
Merge branch 'main' into codex/pcp-piecewise-debug-main
mergify[bot] 5149d19
fix wrong update
pisceskkk File filter
Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
There are no files selected for viewing
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,109 @@ | ||
| # SPDX-License-Identifier: Apache-2.0 | ||
| # SPDX-FileCopyrightText: Copyright contributors to the vLLM project | ||
| import numpy as np | ||
| import pytest | ||
| import torch | ||
|
|
||
| from vllm.v1.worker.gpu import pcp_manager as pcp_manager_module | ||
| from vllm.v1.worker.gpu.pcp_manager import PCPManager | ||
|
|
||
|
|
||
| def _copy_to_cpu(value, out=None, device=None): | ||
| tensor = torch.from_numpy(value) if isinstance(value, np.ndarray) else value | ||
| if out is not None: | ||
| return out.copy_(tensor) | ||
| return tensor | ||
|
|
||
|
|
||
| def test_replicated_decode_piecewise_graph_padding(monkeypatch): | ||
| manager = PCPManager( | ||
| pcp_world_size=2, | ||
| pcp_rank=0, | ||
| device=torch.device("cpu"), | ||
| dcp_world_size=1, | ||
| ) | ||
| monkeypatch.setattr(pcp_manager_module, "async_copy_to_gpu", _copy_to_cpu) | ||
|
|
||
| segments_by_rank, per_rank_num_tokens = manager._build_batch_layout( | ||
| num_scheduled_tokens=np.ones(3, dtype=np.int32), | ||
| num_computed_tokens=np.full(3, 16, dtype=np.int32), | ||
| is_prefilling=np.zeros(3, dtype=np.bool_), | ||
| query_start_loc_np=np.arange(4, dtype=np.int32), | ||
| padded_num_tokens=4, | ||
| ) | ||
|
|
||
| assert per_rank_num_tokens == [3, 3] | ||
| request_indices = [ | ||
| [segment.global_batch_req_idx for segment in rank] for rank in segments_by_rank | ||
| ] | ||
| assert request_indices == [[0, 1, 2], [0, 1, 2]] | ||
| assert torch.equal(manager._hidden_restore_idx, torch.tensor([0, 1, 2])) | ||
| assert torch.equal( | ||
| manager._padded_gather_idx, | ||
| torch.tensor([0, 1, 2, 0, 0, 1, 2, 0]), | ||
| ) | ||
| assert torch.equal( | ||
| manager._gathered_kv_write_mask, | ||
| torch.tensor([True, True, True, False, False, False, False, False]), | ||
| ) | ||
|
|
||
|
|
||
| def test_input_buffers_are_exposed_for_cudagraph_capture(): | ||
| manager = PCPManager( | ||
| pcp_world_size=2, | ||
| pcp_rank=0, | ||
| device=torch.device("cpu"), | ||
| max_num_reqs=4, | ||
| max_num_tokens=8, | ||
| ) | ||
|
|
||
| assert manager.input_buffers is manager._input_buffers | ||
| assert manager.input_buffers.input_ids.shape == (8,) | ||
| assert manager.input_buffers.positions.shape == (8,) | ||
| assert manager.input_buffers.is_padding.shape == (8,) | ||
|
|
||
|
|
||
| @pytest.mark.parametrize( | ||
| ("pcp_world_size", "num_scheduled_tokens", "is_prefilling", "expected"), | ||
| [ | ||
| (2, [8], [True], 4), | ||
| (2, [7], [True], 4), | ||
| (2, [3], [False], 3), | ||
| (2, [3, 8], [False, True], 7), | ||
| (4, [2, 9], [False, True], 5), | ||
| ], | ||
| ) | ||
| def test_num_tokens_for_dispatch_uses_largest_pcp_rank( | ||
| pcp_world_size, num_scheduled_tokens, is_prefilling, expected | ||
| ): | ||
| manager = PCPManager( | ||
| pcp_world_size=pcp_world_size, | ||
| pcp_rank=0, | ||
| device=torch.device("cpu"), | ||
| ) | ||
|
|
||
| actual = manager.get_num_tokens_for_dispatch( | ||
| np.asarray(num_scheduled_tokens, dtype=np.int32), | ||
| np.asarray(is_prefilling, dtype=np.bool_), | ||
| ) | ||
|
|
||
| assert actual == expected | ||
|
|
||
|
|
||
| def test_graph_padding_cannot_be_smaller_than_largest_pcp_rank(monkeypatch): | ||
| manager = PCPManager( | ||
| pcp_world_size=2, | ||
| pcp_rank=0, | ||
| device=torch.device("cpu"), | ||
| dcp_world_size=1, | ||
| ) | ||
| monkeypatch.setattr(pcp_manager_module, "async_copy_to_gpu", _copy_to_cpu) | ||
|
|
||
| with pytest.raises(ValueError, match="smaller than the largest rank-local batch"): | ||
| manager._build_batch_layout( | ||
| num_scheduled_tokens=np.ones(3, dtype=np.int32), | ||
| num_computed_tokens=np.full(3, 16, dtype=np.int32), | ||
| is_prefilling=np.zeros(3, dtype=np.bool_), | ||
| query_start_loc_np=np.arange(4, dtype=np.int32), | ||
| padded_num_tokens=2, | ||
| ) |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
Uh oh!
There was an error while loading. Please reload this page.