Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
45 changes: 45 additions & 0 deletions tests/entrypoints/test_cli_main.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,45 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project

from vllm.entrypoints.cli import main as cli_main
from vllm.utils import system_utils


def test_serve_bootstraps_forkserver_before_command_imports(monkeypatch):
calls: list[tuple[list[str], bool]] = []
monkeypatch.setenv("VLLM_WORKER_MULTIPROC_METHOD", "forkserver")
monkeypatch.setattr("sys.argv", ["vllm", "serve", "model"])
monkeypatch.setattr(
system_utils,
"ensure_cuda_clean_forkserver",
lambda modules, *, set_start_method=False: calls.append(
(modules, set_start_method)
),
)

cli_main._bootstrap_cuda_clean_forkserver()

assert calls == [
(
[
"vllm.v1.engine.async_llm",
"vllm.v1.executor.multiproc_executor",
"vllm.v1.worker.gpu_worker",
],
True,
)
]


def test_non_serve_command_does_not_bootstrap_forkserver(monkeypatch):
monkeypatch.setenv("VLLM_WORKER_MULTIPROC_METHOD", "forkserver")
monkeypatch.setattr("sys.argv", ["vllm", "bench"])
monkeypatch.setattr(
system_utils,
"ensure_cuda_clean_forkserver",
lambda *args, **kwargs: (_ for _ in ()).throw(
AssertionError("non-serve command must not start a forkserver")
),
)

cli_main._bootstrap_cuda_clean_forkserver()
105 changes: 104 additions & 1 deletion tests/utils_/test_system_utils.py
Original file line number Diff line number Diff line change
@@ -1,11 +1,21 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project

import multiprocessing
import multiprocessing.forkserver as forkserver
import os
import tempfile
from pathlib import Path

from vllm.utils.system_utils import _maybe_force_spawn, unique_filepath
import pytest

import vllm.envs as envs
from vllm.utils import system_utils
from vllm.utils.system_utils import (
_maybe_force_spawn,
ensure_cuda_clean_forkserver,
unique_filepath,
)


def test_unique_filepath():
Expand All @@ -25,3 +35,96 @@ def test_numa_bind_forces_spawn(monkeypatch):
monkeypatch.setattr("sys.argv", ["vllm", "serve", "--numa-bind"])
_maybe_force_spawn()
assert os.environ["VLLM_WORKER_MULTIPROC_METHOD"] == "spawn"


def test_worker_multiproc_method_accepts_forkserver(monkeypatch):
monkeypatch.setenv("VLLM_WORKER_MULTIPROC_METHOD", "forkserver")

assert envs.environment_variables["VLLM_WORKER_MULTIPROC_METHOD"]() == "forkserver"


def test_ensure_cuda_clean_forkserver_starts_with_deduplicated_preload(
monkeypatch,
):
calls: dict[str, object] = {}
monkeypatch.setattr(system_utils, "_cuda_clean_forkserver_owner_pid", None)
monkeypatch.setattr(system_utils, "cuda_is_initialized", lambda: False)
monkeypatch.setattr(system_utils, "xpu_is_initialized", lambda: False)
monkeypatch.setattr(
multiprocessing,
"set_start_method",
lambda method, force=False: calls.update(method=(method, force)),
)
monkeypatch.setattr(
multiprocessing,
"set_forkserver_preload",
lambda modules: calls.update(preload=modules),
)
monkeypatch.setattr(
forkserver,
"ensure_running",
lambda: calls.update(running=True),
)

ensure_cuda_clean_forkserver(
["vllm.worker", "vllm.executor", "vllm.worker"],
set_start_method=True,
)

assert calls == {
"method": ("forkserver", True),
"preload": ["vllm.worker", "vllm.executor"],
"running": True,
}
assert system_utils._cuda_clean_forkserver_owner_pid == os.getpid()


def test_ensure_cuda_clean_forkserver_is_idempotent(monkeypatch):
monkeypatch.setattr(system_utils, "_cuda_clean_forkserver_owner_pid", os.getpid())
monkeypatch.setattr(
system_utils,
"cuda_is_initialized",
lambda: pytest.fail("idempotent call must not recheck CUDA"),
)

ensure_cuda_clean_forkserver(["vllm.worker"])


def test_active_cuda_clean_forkserver_is_not_replaced(monkeypatch):
monkeypatch.setenv("VLLM_WORKER_MULTIPROC_METHOD", "forkserver")
monkeypatch.setattr(system_utils, "_cuda_clean_forkserver_owner_pid", os.getpid())
monkeypatch.setattr(system_utils, "cuda_is_initialized", lambda: True)

_maybe_force_spawn()

assert os.environ["VLLM_WORKER_MULTIPROC_METHOD"] == "forkserver"


def test_unstarted_forkserver_is_replaced_after_cuda_init(monkeypatch):
monkeypatch.setenv("VLLM_WORKER_MULTIPROC_METHOD", "forkserver")
monkeypatch.setattr(system_utils, "_cuda_clean_forkserver_owner_pid", None)
monkeypatch.setattr(system_utils, "cuda_is_initialized", lambda: True)
monkeypatch.setattr(system_utils, "xpu_is_initialized", lambda: False)
monkeypatch.setattr(system_utils, "is_in_ray_actor", lambda: False)
monkeypatch.setattr(system_utils, "in_wsl", lambda: False)
monkeypatch.setattr("sys.argv", ["vllm", "serve"])

_maybe_force_spawn()

assert os.environ["VLLM_WORKER_MULTIPROC_METHOD"] == "spawn"


@pytest.mark.parametrize("initialized", ["cuda", "xpu"])
def test_ensure_cuda_clean_forkserver_rejects_initialized_parent(
monkeypatch, initialized
):
monkeypatch.setattr(system_utils, "_cuda_clean_forkserver_owner_pid", None)
monkeypatch.setattr(
system_utils, "cuda_is_initialized", lambda: initialized == "cuda"
)
monkeypatch.setattr(
system_utils, "xpu_is_initialized", lambda: initialized == "xpu"
)

with pytest.raises(RuntimeError, match="after CUDA/XPU initialization"):
ensure_cuda_clean_forkserver(["vllm.worker"])
20 changes: 19 additions & 1 deletion tests/v1/worker/test_gpu_worker.py
Original file line number Diff line number Diff line change
@@ -1,8 +1,9 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project

import sys
from types import SimpleNamespace
from unittest.mock import patch
from unittest.mock import Mock, patch

import pytest

Expand All @@ -22,6 +23,23 @@
)


def test_gpu_worker_keeps_cuda_warmup_import_lazy(monkeypatch: pytest.MonkeyPatch):
# Forkserver preloads this module. Binding kernel_warmup at module scope
# imports CUDA-initializing warmup dependencies into the server snapshot.
assert gpu_worker_module.kernel_warmup.__module__ == gpu_worker_module.__name__

run_kernel_warmup = Mock()
monkeypatch.setitem(
sys.modules,
"vllm.model_executor.warmup.kernel_warmup",
SimpleNamespace(kernel_warmup=run_kernel_warmup),
)
worker = object()
gpu_worker_module.kernel_warmup(worker)

run_kernel_warmup.assert_called_once_with(worker)


def _worker_with_mm_config(
mm_config: SimpleNamespace,
*,
Expand Down
22 changes: 22 additions & 0 deletions vllm/entrypoints/cli/main.py
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@
to avoid certain eager import breakage."""

import importlib.metadata
import os
import sys
from importlib.util import find_spec

Expand All @@ -14,7 +15,28 @@
logger = init_logger(__name__)


def _bootstrap_cuda_clean_forkserver() -> None:
"""Start the serving forkserver before CLI imports can initialize CUDA."""
if len(sys.argv) < 2 or sys.argv[1] != "serve":
return
if os.environ.get("VLLM_WORKER_MULTIPROC_METHOD") != "forkserver":
return

from vllm.utils.system_utils import ensure_cuda_clean_forkserver

ensure_cuda_clean_forkserver(
[
"vllm.v1.engine.async_llm",
"vllm.v1.executor.multiproc_executor",
"vllm.v1.worker.gpu_worker",
],
set_start_method=True,
)


def main():
_bootstrap_cuda_clean_forkserver()

import vllm.entrypoints.cli.benchmark.main
import vllm.entrypoints.cli.collect_env
import vllm.entrypoints.cli.launch
Expand Down
29 changes: 18 additions & 11 deletions vllm/entrypoints/openai/api_server.py
Original file line number Diff line number Diff line change
Expand Up @@ -3,9 +3,6 @@
import asyncio
import importlib
import inspect
import multiprocessing
import multiprocessing.forkserver as forkserver
import os
import signal
import socket
import tempfile
Expand Down Expand Up @@ -66,7 +63,11 @@
from vllm.usage.usage_lib import UsageContext
from vllm.utils.argparse_utils import FlexibleArgumentParser
from vllm.utils.network_utils import is_valid_ipv6_address
from vllm.utils.system_utils import decorate_logs, set_ulimit
from vllm.utils.system_utils import (
decorate_logs,
ensure_cuda_clean_forkserver,
set_ulimit,
)
from vllm.v1.engine.exceptions import EngineDeadError, EngineGenerateError
from vllm.version import __version__ as VLLM_VERSION

Expand Down Expand Up @@ -120,13 +121,19 @@ async def build_async_engine_client(
usage_context: UsageContext = UsageContext.OPENAI_API_SERVER,
client_config: dict[str, Any] | None = None,
) -> AsyncIterator[EngineClient]:
if os.getenv("VLLM_WORKER_MULTIPROC_METHOD") == "forkserver":
# The executor is expected to be mp.
# Pre-import heavy modules in the forkserver process
logger.debug("Setup forkserver with pre-imports")
multiprocessing.set_start_method("forkserver")
multiprocessing.set_forkserver_preload(["vllm.v1.engine.async_llm"])
forkserver.ensure_running()
if envs.VLLM_WORKER_MULTIPROC_METHOD == "forkserver":
# EngineCore is forked from this CUDA-clean server. Preloading the
# executor and worker stack lets its pages be shared instead of imported
# independently by every local rank.
logger.debug("Setup CUDA-clean API forkserver with worker preloads")
ensure_cuda_clean_forkserver(
[
"vllm.v1.engine.async_llm",
"vllm.v1.executor.multiproc_executor",
"vllm.v1.worker.gpu_worker",
],
set_start_method=True,
)
logger.debug("Forkserver setup complete!")

# Context manager to handle engine_client lifecycle
Expand Down
6 changes: 3 additions & 3 deletions vllm/envs.py
Original file line number Diff line number Diff line change
Expand Up @@ -92,7 +92,7 @@
VLLM_USE_RAY_V2_EXECUTOR_BACKEND: bool = False
VLLM_DISTRIBUTED_USE_SPLIT_GROUP: bool = False
VLLM_XLA_USE_SPMD: bool = False
VLLM_WORKER_MULTIPROC_METHOD: Literal["fork", "spawn"] = "fork"
VLLM_WORKER_MULTIPROC_METHOD: Literal["fork", "spawn", "forkserver"] = "fork"
VLLM_ASSETS_CACHE: str = os.path.join(VLLM_CACHE_ROOT, "assets")
VLLM_ASSETS_CACHE_MODEL_CLEAN: bool = False
VLLM_IMAGE_FETCH_TIMEOUT: int = 5
Expand Down Expand Up @@ -949,9 +949,9 @@ def _resolve_rust_frontend_path() -> str | None:
int(os.getenv("VLLM_DISTRIBUTED_USE_SPLIT_GROUP", "0"))
),
# Use dedicated multiprocess context for workers.
# Both spawn and fork work
# spawn, fork, and an explicitly configured forkserver are supported.
"VLLM_WORKER_MULTIPROC_METHOD": env_with_choices(
"VLLM_WORKER_MULTIPROC_METHOD", "fork", ["spawn", "fork"]
"VLLM_WORKER_MULTIPROC_METHOD", "fork", ["spawn", "fork", "forkserver"]
),
# Path to the cache for storing downloaded assets
"VLLM_ASSETS_CACHE": lambda: os.path.expanduser(
Expand Down
46 changes: 45 additions & 1 deletion vllm/utils/system_utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -112,6 +112,47 @@ def unique_filepath(fn: Callable[[int], Path]) -> Path:
# Process management utilities


_cuda_clean_forkserver_owner_pid: int | None = None


def _cuda_clean_forkserver_is_running() -> bool:
return _cuda_clean_forkserver_owner_pid == os.getpid()


def ensure_cuda_clean_forkserver(
preload_modules: list[str],
*,
set_start_method: bool = False,
) -> None:
"""Start a forkserver before this process initializes an accelerator.

A forkserver child inherits the server's runtime state. Starting that
server after CUDA/XPU initialization would reproduce the same unsafe state
inheritance as a direct fork, so fail closed instead of creating workers
that can fail later during device initialization.
"""
global _cuda_clean_forkserver_owner_pid

if _cuda_clean_forkserver_is_running():
return
if cuda_is_initialized() or xpu_is_initialized():
raise RuntimeError(
"Cannot start the vLLM forkserver after CUDA/XPU initialization. "
"Use VLLM_WORKER_MULTIPROC_METHOD=spawn instead."
)
if not preload_modules:
raise ValueError("forkserver preload_modules must not be empty")

if set_start_method:
multiprocessing.set_start_method("forkserver", force=True)
multiprocessing.set_forkserver_preload(list(dict.fromkeys(preload_modules)))

import multiprocessing.forkserver as forkserver

forkserver.ensure_running()
_cuda_clean_forkserver_owner_pid = os.getpid()


def _sync_visible_devices_env_vars():
"""Sync HIP/CUDA visibility env vars before spawning (ROCm only)."""

Expand All @@ -127,7 +168,10 @@ def _maybe_force_spawn():
"""Check if we need to force the use of the `spawn` multiprocessing start
method.
"""
if os.environ.get("VLLM_WORKER_MULTIPROC_METHOD") == "spawn":
method = os.environ.get("VLLM_WORKER_MULTIPROC_METHOD")
if method == "spawn" or (
method == "forkserver" and _cuda_clean_forkserver_is_running()
):
return

reasons = []
Expand Down
Loading
Loading