Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
24 commits
Select commit Hold shift + click to select a range
30a11dc
build(vllm): bump core to v0.32.0
JulienDarve Oct 18, 2026
d1c89fa
chore(vllm): merge main and regenerate version tables [skip ci]
JulienDarve Oct 19, 2026
927fef2
build(vllm): bump Omni to v0.32.0rc1
JulienDarve Oct 22, 2026
619db18
docs: preserve attribution without unavailable profile link
JulienDarve Oct 22, 2026
61729d6
test(vllm): update Omni 0.32 contracts and bound listener test
JulienDarve Oct 22, 2026
16fac6c
fix(vllm): align token budgets and image normalization with 0.32
JulienDarve Oct 22, 2026
7cb4a96
fix(vllm): type listener tuples and simplify version tests
JulienDarve Oct 22, 2026
9b60679
build(vllm): keep the upstream Transformers stack on CUDA
JulienDarve Oct 23, 2026
c15cc7a
fix(vllm): import the chat-template helper from vLLM in the realtime …
JulienDarve Oct 23, 2026
b7b0631
test(vllm): check token-budget policy only in its owning test
JulienDarve Oct 23, 2026
5964041
refactor(vllm): drop pre-0.28 engine-launch tuple support
JulienDarve Oct 23, 2026
97ff31a
Merge main into jdarve/vllm-bump-v0.32.0
JulienDarve Oct 23, 2026
ec13de6
test(vllm): restore the sampling max-tokens test from main
JulienDarve Oct 23, 2026
c5f238e
test(vllm): expect only the shipped engine-core struct fields
JulienDarve Oct 23, 2026
f3b1ea8
docs(vllm): note when the output-clamping version check can go
JulienDarve Oct 23, 2026
0c69441
test(vllm): check the vLLM 0.32 max_tokens cap on the CUDA image
JulienDarve Oct 23, 2026
b7da936
test(vllm): run the listener child from a function instead of a string
JulienDarve Oct 23, 2026
93d2b08
refactor(vllm): compute max_tokens with vLLM's helper on every image
JulienDarve Oct 23, 2026
8c29bda
refactor(vllm): import launch_core_engines at module level
JulienDarve Oct 23, 2026
fb7be35
test: stub vLLM's serve utils for marker-only collection
JulienDarve Oct 23, 2026
8d8b4c5
fix(vllm): reject max_tokens above max_model_len like vLLM 0.32
JulienDarve Oct 23, 2026
2797623
build(vllm): match vLLM 0.32's NIXL 1.5.0 in the extra and docs
JulienDarve Oct 23, 2026
ffa2ac9
build(vllm): keep pydantic within ai-dynamo-runtime's range on CUDA
JulienDarve Oct 23, 2026
35935dc
Merge main into jdarve/vllm-bump-v0.32.0
JulienDarve Oct 23, 2026
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 6 additions & 2 deletions components/src/dynamo/frontend/prepost.py
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,6 @@
from functools import lru_cache, partial
from typing import TYPE_CHECKING, Any, Protocol, TypeGuard, cast

from vllm.entrypoints.chat_utils import make_tool_call_id
from vllm.entrypoints.openai.chat_completion.protocol import (
ChatCompletionNamedFunction,
ChatCompletionNamedToolChoiceParam,
Expand All @@ -29,7 +28,12 @@
from vllm.utils.async_utils import make_async

from dynamo.common.utils.guided_json import admits_only_empty_object
from dynamo.frontend.vllm_protocol import DeltaFunctionCall, DeltaMessage, DeltaToolCall
from dynamo.frontend.vllm_protocol import (
DeltaFunctionCall,
DeltaMessage,
DeltaToolCall,
make_tool_call_id,
)
from dynamo.llm.exceptions import InvalidArgument

from .structural_tag_policy import (
Expand Down
2 changes: 1 addition & 1 deletion components/src/dynamo/frontend/vllm_processor.py
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,6 @@

from msgspec.structs import replace as msgspec_replace
from vllm.config import CacheConfig, LoadConfig, ModelConfig, VllmConfig
from vllm.entrypoints.chat_utils import load_chat_template
from vllm.exceptions import VLLMClientError
from vllm.reasoning import ReasoningParser, ReasoningParserManager
from vllm.sampling_params import RequestOutputKind, SamplingParams
Expand Down Expand Up @@ -62,6 +61,7 @@
random_uuid,
resolve_chat_template,
)
from .vllm_protocol import load_chat_template

logger = logging.getLogger(__name__)

Expand Down
10 changes: 9 additions & 1 deletion components/src/dynamo/frontend/vllm_protocol.py
Original file line number Diff line number Diff line change
@@ -1,10 +1,16 @@
# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
# SPDX-License-Identifier: Apache-2.0

"""Compatibility imports for vLLM's shared OpenAI protocol types."""
"""Compatibility imports for vLLM's frontend helpers and OpenAI protocol types."""

from importlib.util import find_spec

# vLLM 0.32 moved the chat helpers to vllm.renderers.
Comment thread
JulienDarve marked this conversation as resolved.
Resolved
if find_spec("vllm.renderers.chat_utils") is not None:
Comment thread
JulienDarve marked this conversation as resolved.
Resolved
from vllm.renderers.chat_utils import load_chat_template, make_tool_call_id
else:
from vllm.entrypoints.chat_utils import load_chat_template, make_tool_call_id

try:
_HAS_GENERATE_PROTOCOL = (
find_spec("vllm.entrypoints.generate.base.protocol") is not None
Expand Down Expand Up @@ -34,4 +40,6 @@
"DeltaMessage",
"DeltaToolCall",
"FunctionDefinition",
"load_chat_template",
"make_tool_call_id",
]
12 changes: 11 additions & 1 deletion components/src/dynamo/vllm/capacity.py
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,16 @@
logger = logging.getLogger(__name__)


def has_vllm_output_clamping() -> bool:
"""vLLM 0.32 caps oversized output budgets instead of rejecting them.

Remove when the CPU and XPU images reach vLLM 0.32.
Comment thread
JulienDarve marked this conversation as resolved.
Resolved
"""
from vllm.version import __version_tuple__

return __version_tuple__[:2] >= (0, 32)


def publish_vllm_token_budget(runtime_config: Any, max_model_len: int | None) -> None:
"""Publish vLLM's request-overflow contract to the Dynamo frontend."""
if max_model_len is None:
Expand All @@ -21,7 +31,7 @@ def publish_vllm_token_budget(runtime_config: Any, max_model_len: int | None) ->
TokenBudget(
combined_limit=max_model_len,
reject_prompt_overflow=True,
reject_total_overflow=True,
reject_total_overflow=not has_vllm_output_clamping(),
),
)

Expand Down
129 changes: 72 additions & 57 deletions components/src/dynamo/vllm/embedding_worker_processes.py
Original file line number Diff line number Diff line change
Expand Up @@ -13,11 +13,11 @@
from __future__ import annotations

import copy
import importlib
import json
import logging
import os
import signal
import socket
import subprocess
import sys
import tempfile
Expand All @@ -29,42 +29,38 @@
import vllm
from vllm.config import VllmConfig
from vllm.usage.usage_lib import UsageContext
from vllm.v1.engine import utils as engine_utils
from vllm.v1.engine.async_llm import AsyncLLM
from vllm.v1.engine.utils import launch_core_engines
from vllm.v1.executor import Executor

# Resolved on first use, not at import: vLLM renamed get_engine_zmq_addresses in
# vllm-project/vllm#54113, and this module is imported by every dynamo.vllm worker
# while only --embedding-worker-processes > 1 calls into it. Module attributes so
# tests can substitute them.
get_engine_zmq_addresses: Callable[..., Any] | None = None
launch_core_engines: Callable[..., Any] | None = None


def _resolve_engine_utils() -> tuple[Callable[..., Any], Callable[..., Any]]:
"""Bind the vLLM engine-launch helpers this pool needs, failing with a clear message.

Returns ``(get_engine_zmq_addresses, launch_core_engines)`` so callers hold
non-optional references; the module attributes are kept in sync for tests.
"""
global get_engine_zmq_addresses, launch_core_engines
engine_utils = importlib.import_module("vllm.v1.engine.utils")

if launch_core_engines is None:
launch_core_engines = engine_utils.launch_core_engines
if get_engine_zmq_addresses is None:
resolved = getattr(engine_utils, "get_engine_zmq_addresses", None)
if resolved is None:
# vllm#54113: the supervisor now binds the frontend listeners up front and
# children adopt them; this pool still lets children bind the addresses
# themselves, so refuse rather than hand out addresses that are already bound.
raise RuntimeError(
"--embedding-worker-processes > 1 is not yet supported on this vLLM: "
f"vLLM {vllm.__version__} has no get_engine_zmq_addresses (it binds "
"frontend listeners in the supervisor since vllm-project/vllm#54113). "
"Run with --embedding-worker-processes 1 until the pool adopts inherited listeners."
)
get_engine_zmq_addresses = resolved
return get_engine_zmq_addresses, launch_core_engines
def _resolve_engine_utils(
vllm_config: VllmConfig, process_count: int
) -> tuple[Any, Any]:
# vLLM 0.32 replaces address allocation with supervisor-bound listeners.
get_addresses = getattr(engine_utils, "get_engine_zmq_addresses", None)
if get_addresses is not None:
return get_addresses(vllm_config, process_count), None
bind_listeners = getattr(engine_utils, "bind_engine_zmq_listeners", None)
if bind_listeners is None:
raise RuntimeError(
f"vLLM {vllm.__version__} provides neither get_engine_zmq_addresses "
"nor bind_engine_zmq_listeners"
)
listeners = bind_listeners(vllm_config, process_count)
return listeners.addresses, listeners


def _client_listeners(
zmq_listeners: Any, process_index: int
) -> tuple[socket.socket, ...]:
if zmq_listeners is None:
return ()
return (
zmq_listeners.inputs[process_index].socket,
zmq_listeners.outputs[process_index].socket,
)


logger = logging.getLogger(__name__)
Expand All @@ -73,12 +69,11 @@ def _resolve_engine_utils() -> tuple[Callable[..., Any], Callable[..., Any]]:
_INDEX_ENV = "DYN_VLLM_EMBEDDING_PROCESS_INDEX"
_PARENT_PID_ENV = "DYN_VLLM_EMBEDDING_PARENT_PID"
_ENGINE_ADDRESSES_ENV = "DYN_VLLM_EMBEDDING_ENGINE_ADDRESSES"
_LISTENER_FDS_ENV = "DYN_VLLM_EMBEDDING_LISTENER_FDS"
_CHILD_ROLE = "child"
_RPC_BASE_PATH_ENV = "VLLM_RPC_BASE_PATH"
# The two shapes _unpack_core_engine_launch accepts were read from these vLLM
# releases: 0.27.1, which container/context.yaml builds the XPU images on, and
# 0.28.0, the pin in pyproject.toml. Re-check that helper when either moves.
_VERIFIED_VLLM_VERSIONS = ("0.27.1", "0.28.0")
# Re-check the launch shape when either supported pin moves.
_VERIFIED_VLLM_VERSIONS = ("0.31.0", "0.32.0")


def is_embedding_process_child() -> bool:
Expand Down Expand Up @@ -147,22 +142,25 @@ def _attach_client(
stat_loggers: list[Any],
enable_log_requests: bool,
disable_log_stats: bool,
listeners: tuple[socket.socket, ...] = (),
) -> tuple[AsyncLLM, VllmConfig]:
client_vllm_config = _client_config(
vllm_config,
process_count=process_count,
process_index=process_index,
)
client_addresses: dict[str, Any] = {
"input_address": input_address,
"output_address": output_address,
}
client_addresses.update(zip(("input_listener", "output_listener"), listeners))
client = AsyncLLM.from_vllm_config(
vllm_config=client_vllm_config,
usage_context=usage_context,
stat_loggers=stat_loggers,
enable_log_requests=enable_log_requests,
disable_log_stats=disable_log_stats,
client_addresses={
"input_address": input_address,
"output_address": output_address,
},
client_addresses=client_addresses,
client_count=process_count,
client_index=process_index,
)
Expand Down Expand Up @@ -223,12 +221,17 @@ def _child_environment(
process_index: int,
addresses_json: str,
parent_pid: int,
listener_fds: tuple[int, ...] = (),
) -> dict[str, str]:
env = os.environ.copy()
env[_ROLE_ENV] = _CHILD_ROLE
env[_INDEX_ENV] = str(process_index)
env[_PARENT_PID_ENV] = str(parent_pid)
env[_ENGINE_ADDRESSES_ENV] = addresses_json
if listener_fds:
env[_LISTENER_FDS_ENV] = json.dumps(listener_fds)
else:
env.pop(_LISTENER_FDS_ENV, None)
env["PYTHONUNBUFFERED"] = "1"

# Each Dynamo runtime process needs its own system-status listener, but the
Expand Down Expand Up @@ -304,12 +307,14 @@ def __init__(
engine_manager: Any,
rpc_directory: tempfile.TemporaryDirectory,
previous_rpc_base_path: str | None,
zmq_listeners: Any = None,
parent_failure_callback: Callable[[int, int], None] | None = None,
) -> None:
self.children = children
self.engine_manager = engine_manager
self.rpc_directory = rpc_directory
self.previous_rpc_base_path = previous_rpc_base_path
self.zmq_listeners = zmq_listeners
self._parent_failure_callback = (
parent_failure_callback or self._terminate_parent
)
Expand Down Expand Up @@ -359,6 +364,9 @@ def cleanup(self, timeout: float = 10.0) -> None:
except Exception:
logger.exception("Failed to shut down shared embedding EngineCore")

if self.zmq_listeners is not None:
self.zmq_listeners.cleanup()

try:
self.rpc_directory.cleanup()
except Exception:
Expand Down Expand Up @@ -424,19 +432,7 @@ def _unrecognized_launch_shape(detail: str) -> RuntimeError:


def _unpack_core_engine_launch(launch: Any) -> tuple[Any, Any, Any, Any]:
"""Normalize what ``launch_core_engines`` yields across vLLM releases.

vLLM 0.28 and later yield a ``CoreEngineLaunch`` object defining no
``__iter__``; earlier releases yield a plain 4-tuple. Duck-typing the
yielded object avoids importing ``CoreEngineLaunch``, a name that does not
exist before 0.28.
"""
if isinstance(launch, tuple):
if len(launch) != 4:
raise _unrecognized_launch_shape(
f"a {len(launch)}-tuple rather than the expected 4-tuple"
)
return launch
"""Read the fields of vLLM's ``CoreEngineLaunch``, which is not iterable."""
try:
return (
launch.engine_manager,
Expand Down Expand Up @@ -465,6 +461,14 @@ def create_shared_embedding_engine_client(

if is_embedding_process_child():
process_index, inputs, outputs = _decode_child_addresses(process_count)
listener_fds = json.loads(os.environ.get(_LISTENER_FDS_ENV, "null"))
listeners: tuple[socket.socket, ...] = ()
if listener_fds is not None:
input_fd, output_fd = listener_fds
listeners = (
socket.socket(fileno=input_fd),
socket.socket(fileno=output_fd),
)
client, client_config = _attach_client(
vllm_config,
process_count=process_count,
Expand All @@ -475,6 +479,7 @@ def create_shared_embedding_engine_client(
stat_loggers=stat_loggers,
enable_log_requests=enable_log_requests,
disable_log_stats=disable_log_stats,
listeners=listeners,
)
logger.info(
"Attached Dynamo embedding child %d/%d to shared EngineCore",
Expand All @@ -493,13 +498,13 @@ def create_shared_embedding_engine_client(

engine_manager = None
process_group = None
zmq_listeners = None
children: list[tuple[int, subprocess.Popen]] = []
try:
vllm_config.parallel_config._api_process_count = process_count
vllm_config.parallel_config._api_process_rank = -1
executor_class = Executor.get_class(vllm_config)
engine_zmq_addresses, core_engines_launcher = _resolve_engine_utils()
addresses = engine_zmq_addresses(vllm_config, process_count)
addresses, zmq_listeners = _resolve_engine_utils(vllm_config, process_count)

parent_logger_config = _client_config(
vllm_config,
Expand All @@ -512,7 +517,7 @@ def create_shared_embedding_engine_client(
with ThreadPoolExecutor(
max_workers=1, thread_name_prefix="dynamo-embedding-client"
) as client_pool:
with core_engines_launcher(
with launch_core_engines(
vllm_config,
executor_class,
not disable_log_stats,
Expand All @@ -539,22 +544,29 @@ def create_shared_embedding_engine_client(
)
command = [sys.executable, "-m", "dynamo.vllm", *sys.argv[1:]]
for process_index in range(1, process_count):
child_listeners = _client_listeners(zmq_listeners, process_index)
listener_fds = tuple(s.fileno() for s in child_listeners)
child = subprocess.Popen(
command,
env=_child_environment(
process_count=process_count,
process_index=process_index,
addresses_json=addresses_json,
parent_pid=os.getpid(),
listener_fds=listener_fds,
),
pass_fds=listener_fds,
)
children.append((process_index, child))
for listener in child_listeners:
listener.close()

process_group = EmbeddingWorkerProcessGroup(
children=children,
engine_manager=engine_manager,
rpc_directory=rpc_directory,
previous_rpc_base_path=previous_rpc_base_path,
zmq_listeners=zmq_listeners,
)
process_group.start_monitor()
parent_client_future = client_pool.submit(
Expand All @@ -568,6 +580,7 @@ def create_shared_embedding_engine_client(
stat_loggers=parent_stat_loggers,
enable_log_requests=enable_log_requests,
disable_log_stats=disable_log_stats,
listeners=_client_listeners(zmq_listeners, 0),
)

parent_client, parent_config = parent_client_future.result()
Expand All @@ -589,6 +602,8 @@ def create_shared_embedding_engine_client(
logger.exception(
"Failed to shut down EngineCore after embedding pool startup error"
)
if zmq_listeners is not None:
zmq_listeners.cleanup()
rpc_directory.cleanup()
if previous_rpc_base_path is None:
os.environ.pop(_RPC_BASE_PATH_ENV, None)
Expand Down
Loading
Loading