Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
11 changes: 6 additions & 5 deletions vllm/v1/fault_tolerance/engine_core_sentinel.py
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""EngineCoreSentinel and fault_tolerant_wrapper for the engine core."""

import json
import threading
from collections.abc import Callable
from typing import TYPE_CHECKING
Expand Down Expand Up @@ -122,17 +123,17 @@ def _reinit_dp_group(self) -> dict:
return {}

parallel_config = engine.vllm_config.parallel_config
worker_key = f"ft_worker_dp_port_{self._dp_reinit_epoch}"
worker_key = f"ft_worker_dp_ports_{self._dp_reinit_epoch}"
engine_key = f"ft_engine_dp_port_{self._dp_reinit_epoch}"
self._dp_reinit_epoch += 1

if parallel_config.data_parallel_rank == 0:
worker_port = get_open_port()
worker_ports = [get_open_port() for _ in range(parallel_config.world_size)]
engine_port = get_open_port()
engine.dp_store.set(worker_key, str(worker_port).encode())
engine.dp_store.set(worker_key, json.dumps(worker_ports).encode())
engine.dp_store.set(engine_key, str(engine_port).encode())
else:
worker_port = int(engine.dp_store.get(worker_key).decode())
worker_ports = json.loads(engine.dp_store.get(worker_key).decode())
engine_port = int(engine.dp_store.get(engine_key).decode())
Comment on lines 130 to 137

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

critical

Generating world_size ports is redundant and incorrect. In a multi-TP/PP setup, the data parallel group is formed by ranks that share the same TP/PP/PCP indices. Therefore, there are world_size // data_parallel_size independent DP groups, and we only need one port per DP group. Generating world_size ports and indexing them by self.worker.rank will cause different ranks in the same DP group to use different ports, preventing them from connecting and causing initialization to hang.

Suggested change
if parallel_config.data_parallel_rank == 0:
worker_port = get_open_port()
worker_ports = [get_open_port() for _ in range(parallel_config.world_size)]
engine_port = get_open_port()
engine.dp_store.set(worker_key, str(worker_port).encode())
engine.dp_store.set(worker_key, json.dumps(worker_ports).encode())
engine.dp_store.set(engine_key, str(engine_port).encode())
else:
worker_port = int(engine.dp_store.get(worker_key).decode())
worker_ports = json.loads(engine.dp_store.get(worker_key).decode())
engine_port = int(engine.dp_store.get(engine_key).decode())
if parallel_config.data_parallel_rank == 0:
num_workers_per_dp_rank = parallel_config.world_size // parallel_config.data_parallel_size
worker_ports = [get_open_port() for _ in range(num_workers_per_dp_rank)]
engine_port = get_open_port()
engine.dp_store.set(worker_key, json.dumps(worker_ports).encode())
engine.dp_store.set(engine_key, str(engine_port).encode())
else:
worker_ports = json.loads(engine.dp_store.get(worker_key).decode())
engine_port = int(engine.dp_store.get(engine_key).decode())


stateless_destroy_torch_distributed_process_group(engine.dp_group)
Expand All @@ -146,7 +147,7 @@ def _reinit_dp_group(self) -> dict:
return_store=True,
)
)
return {"new_stateless_dp_group_port": worker_port}
return {"new_stateless_dp_group_ports": worker_ports}


def fault_tolerant_wrapper(busy_loop_func: Callable):
Expand Down
3 changes: 2 additions & 1 deletion vllm/v1/worker/sentinel/gpu_worker_sentinel.py
Original file line number Diff line number Diff line change
Expand Up @@ -55,7 +55,8 @@ def retry(self, ft_request: FaultToleranceRequest):
if self.dp_size > 1:
old_cpu_group = get_dp_group().cpu_group
stateless_destroy_torch_distributed_process_group(old_cpu_group)
port = params["new_stateless_dp_group_port"]
world_size = self.worker.parallel_config.world_size
port = params["new_stateless_dp_group_ports"][self.worker.rank % world_size]
Comment on lines +58 to +59

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

critical

Using self.worker.rank % world_size resolves to self.worker.rank because the rank is always less than the world size. This assigns a unique port to every single worker. However, workers belonging to the same DP group must share the same port to initialize their process group. Indexing by self.worker.rank % num_workers_per_dp_rank ensures that all ranks in the same DP group get the same port.

            world_size = self.worker.parallel_config.world_size
            num_workers_per_dp_rank = world_size // self.dp_size
            port = params["new_stateless_dp_group_ports"][self.worker.rank % num_workers_per_dp_rank]

get_dp_group().cpu_group = stateless_init_torch_distributed_process_group(
self.data_parallel_master_ip,
port,
Expand Down
Loading