Skip to content
Merged
Show file tree
Hide file tree
Changes from 2 commits
Commits
Show all changes
34 commits
Select commit Hold shift + click to select a range
fb971a6
[Elastic EP] Support Model Runner V2
almogtavor Aug 26, 2026
533aaf5
Merge branch 'main' into mrv2-elastic-ep
almogtavor Sep 6, 2026
c932a47
register eplb on elastic ep dummy load so new workers join the commun…
almogtavor Sep 6, 2026
f1670f5
skip the request level warmup when elastic ep re-warms after a scale
almogtavor Sep 6, 2026
ca0ca71
trim the model runner and elastic execute comments
almogtavor Sep 6, 2026
3abc725
[Elastic EP] Isolate MRV2 warmup state instead of skipping the warmup
almogtavor Sep 7, 2026
1e3378f
Merge branch 'main' into mrv2-elastic-ep
almogtavor Sep 10, 2026
f0fad46
Support elastic EP warmup with reserved slots
almogtavor Sep 11, 2026
3b98fa0
Merge branch 'main' into mrv2-elastic-ep
almogtavor Sep 11, 2026
4119d7b
refactor: improve warmup handling by removing reserved slot configura…
almogtavor Sep 11, 2026
1755262
Merge branch 'main' into mrv2-elastic-ep
almogtavor Sep 11, 2026
fa23c77
refactor: enhance request handling and warmup conditions in GPU model…
almogtavor Sep 11, 2026
6ef02bb
refactor: improve logging for request draining in AsyncLLM
almogtavor Sep 11, 2026
babb214
Drop unrelated whitespace, formatting and comment changes in worker a…
almogtavor Sep 11, 2026
0b3f7dc
Explain the null block redirect flag on BlockTables
almogtavor Sep 11, 2026
2921bd9
Move the dp property comment inside each property and remove dead war…
almogtavor Sep 11, 2026
0097800
Move the MRV2 warmup context manager onto EPLBController and drop the…
almogtavor Sep 11, 2026
8b5bfee
Drop the AsyncLLM admission lock and register parallel sampling child…
almogtavor Sep 11, 2026
c1f8042
Merge branch 'main' into mrv2-elastic-ep
almogtavor Sep 12, 2026
fbea15a
Move preserve_serving_state onto the MRV2 model runner
almogtavor Sep 13, 2026
ca7b124
Suppress EPLB stepping during Elastic EP warmup and drop mocked tests
almogtavor Sep 15, 2026
812c0f2
[Elastic EP] Drain MRV2 requests with the engine-side wait pause
almogtavor Sep 18, 2026
c80da5d
Merge remote-tracking branch 'upstream/main' into mrv2-elastic-ep
almogtavor Sep 18, 2026
b4de0a5
[Elastic EP] Document the drain assumption of preserve_serving_state
almogtavor Sep 19, 2026
2b339b0
Update vllm/v1/engine/core_client.py
almogtavor Sep 23, 2026
8657192
Merge remote-tracking branch 'upstream/main' into mrv2-elastic-ep
almogtavor Sep 24, 2026
85e9725
[Elastic EP] Reset the MRV2 speculator when releasing CUDA graphs
almogtavor Sep 24, 2026
a9ca522
Update vllm/v1/worker/gpu/model_runner.py
almogtavor Sep 25, 2026
8db0a00
[Elastic EP] Move the MRV2 warmup isolation into eplb_utils
almogtavor Sep 25, 2026
1029423
[Elastic EP] Type the model runner and model config args in eplb_utils
almogtavor Sep 28, 2026
c714a73
Merge branch 'main' into mrv2-elastic-ep
njhill Sep 28, 2026
2dbd4cb
Merge branch 'main' into mrv2-elastic-ep
njhill Sep 28, 2026
47b5df5
[Tests] Fix test_speculator_capture_preserves_decode_query_bounds for…
njhill Sep 28, 2026
84c1899
Merge branch 'main' into mrv2-elastic-ep
njhill Sep 28, 2026
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 0 additions & 3 deletions vllm/config/vllm.py
Original file line number Diff line number Diff line change
Expand Up @@ -2593,9 +2593,6 @@ def _get_v2_model_runner_unsupported_features(self) -> list[str]:
if self.parallel_config.use_ubatching:
unsupported.extend(self._get_dbo_unsupported_features())

if self.parallel_config.enable_elastic_ep:
unsupported.append("elastic expert parallelism")

has_logitsproc_plugins = False
if model_config is not None:
from importlib.metadata import entry_points
Expand Down
38 changes: 25 additions & 13 deletions vllm/distributed/elastic_ep/elastic_execute.py
Original file line number Diff line number Diff line change
Expand Up @@ -400,7 +400,13 @@ def _commit_staged_moe_quant_methods(self) -> None:
self._staged_moe_quant_methods.clear()

def _release_cuda_graphs(self) -> None:
if isinstance(self.worker.model_runner.model, CUDAGraphWrapper):
manager = getattr(self.worker.model_runner, "cudagraph_manager", None)
if manager is not None:
# MRV2 captures through CudaGraphManager instead of wrapping the
# model, so neither wrapper branch below ever fires.
manager.release_graphs()

elif isinstance(self.worker.model_runner.model, CUDAGraphWrapper):
wrapper = self.worker.model_runner.model
wrapper.concrete_cudagraph_entries = {}

Expand Down Expand Up @@ -714,14 +720,19 @@ def warm_and_capture(self) -> None:
# (gpu_model_runner.py:3663), which deadlocks if any rank skips it.

# Save and clear block tables so the dummy MoE forward doesn't
# write dummy slot mappings into real KV-cache blocks.
multi_block_table = self.worker.model_runner.input_batch.block_table
# write dummy slot mappings into real KV-cache blocks. MRV2 needs no
# such guard: its dummy runs go through prepare_dummy_attn(), which
# reads BlockTables.input_block_tables (the per-step gather
# destination) and leaves the persistent per-request block ids alone.
multi_block_table = None
saved_block_tables: list[tuple[torch.Tensor, torch.Tensor]] = []
for bt in multi_block_table.block_tables:
saved_block_tables.append(
(bt.block_table.gpu.clone(), bt.block_table.cpu.clone())
)
multi_block_table.clear()
if not self.worker.use_v2_model_runner:
multi_block_table = self.worker.model_runner.input_batch.block_table
for bt in multi_block_table.block_tables:
saved_block_tables.append(
(bt.block_table.gpu.clone(), bt.block_table.cpu.clone())
)
multi_block_table.clear()

# _ensure_workspace_size allocates a fresh tensor on grow, leaving
# any captured CUDA graph with a stale data pointer; drop graphs
Expand All @@ -740,8 +751,9 @@ def warm_and_capture(self) -> None:

lock_workspace()

for bt, (saved_gpu, saved_cpu) in zip(
multi_block_table.block_tables, saved_block_tables
):
bt.block_table.gpu.copy_(saved_gpu)
bt.block_table.cpu.copy_(saved_cpu)
if multi_block_table is not None:
for bt, (saved_gpu, saved_cpu) in zip(
multi_block_table.block_tables, saved_block_tables
):
bt.block_table.gpu.copy_(saved_gpu)
bt.block_table.cpu.copy_(saved_cpu)
18 changes: 17 additions & 1 deletion vllm/v1/worker/gpu/cudagraph_utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -153,7 +153,6 @@ def __init__(
self.decode_query_len = decode_query_len
self.varlen_decode = varlen_decode

self.dp_size = vllm_config.parallel_config.data_parallel_size
self.tp_size = vllm_config.parallel_config.tensor_parallel_size
self.is_first_pp_rank = get_pp_group().is_first_rank
self.is_last_pp_rank = get_pp_group().is_last_rank
Expand Down Expand Up @@ -336,6 +335,23 @@ def _init_candidates(self) -> None:
self._candidates.setdefault(key, []).extend(matching)
current_range_start = num_tokens + 1

@property
def dp_size(self) -> int:
# Not cached: elastic EP rewrites parallel_config in place on scale.
return self.vllm_config.parallel_config.data_parallel_size

def release_graphs(self) -> None:
"""Drop the captured graphs so a later capture() can refill them.

Elastic EP reallocates the MoE workspace when it grows, which leaves
every captured graph holding a stale data pointer. `_capture_descs` is
kept, so `needs_capture()` still reports the work to redo.
"""
self.graphs.clear()
self._graphs_captured = False
if self.breakable_cg_runner is not None:
BreakableCUDAGraphWrapper.clear_all_graphs()

def needs_capture(self) -> bool:
return len(self._capture_descs) > 0

Expand Down
15 changes: 12 additions & 3 deletions vllm/v1/worker/gpu/model_runner.py
Original file line number Diff line number Diff line change
Expand Up @@ -232,9 +232,10 @@ def __init__(self, vllm_config: VllmConfig, device: torch.device):
# Persistent buffer for intermediate tensors (non-first PP ranks).
self.intermediate_tensors: IntermediateTensors | None = None

# Data parallelism.
self.dp_size = self.parallel_config.data_parallel_size
self.dp_rank = self.parallel_config.data_parallel_rank
# Data parallelism. Read through parallel_config (see the dp_size and
# dp_rank properties): elastic EP rewrites it in place on every
# reconfigure, and a cached copy would leave this rank sizing its DP
# collectives for the old world.
Comment thread
almogtavor marked this conversation as resolved.
Resolved

# Dual batch overlap. Created in initialize_kv_cache(), once everything
# it runs the microbatched forward with exists.
Expand Down Expand Up @@ -508,6 +509,14 @@ def load_model(self, load_dummy_weights: bool = False, *args, **kwargs) -> None:

get_offloader().post_init()

@property
def dp_size(self) -> int:
return self.parallel_config.data_parallel_size

@property
def dp_rank(self) -> int:
return self.parallel_config.data_parallel_rank
Comment thread
almogtavor marked this conversation as resolved.
Resolved

def get_model(self) -> nn.Module:
return self.model

Expand Down
Loading