Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 4 additions & 3 deletions components/src/dynamo/vllm/args.py
Original file line number Diff line number Diff line change
Expand Up @@ -237,9 +237,7 @@ def update_engine_config_with_dynamo(
if _uses_nixl_connector(engine_config):
ensure_side_channel_host()

defaults = {
# vLLM 0.13+ renamed 'task' to 'runner'
"runner": "generate",
defaults: Dict[str, Any] = {
# As of vLLM >=0.10.0 the engine unconditionally calls
# `sampling_params.update_from_tokenizer(...)`, so we can no longer
# skip tokenizer initialisation. Setting this to **False** avoids
Expand All @@ -249,6 +247,9 @@ def update_engine_config_with_dynamo(
"disable_log_stats": False,
}

if hasattr(engine_config, "runner"):
logger.debug(f"Using runner={engine_config.runner} from engine args")

kv_cfg = create_kv_events_config(dynamo_config, engine_config)
defaults["kv_events_config"] = kv_cfg
dynamo_config.use_kv_events = kv_cfg is not None and kv_cfg.enable_kv_cache_events
Expand Down
81 changes: 81 additions & 0 deletions components/src/dynamo/vllm/tests/test_vllm_unit.py
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@
ensure_side_channel_host,
get_host_ip,
parse_args,
update_engine_config_with_dynamo,
)
from dynamo.vllm.constants import DisaggregationMode
from dynamo.vllm.tests.conftest import make_cli_args_fixture
Expand Down Expand Up @@ -738,3 +739,83 @@ def test_build_sampling_params_maps_max_thinking_tokens():
}
sp = build_sampling_params(request, default_sampling_params={})
assert sp.thinking_token_budget == 1024


def _make_dynamo_config(**overrides):
"""Build a minimal fake DynamoConfig for update_engine_config_with_dynamo tests."""
defaults = {
"disaggregation_mode": DisaggregationMode.AGGREGATED,
"use_kv_events": False,
"enable_local_indexer": True,
"benchmark_mode": None,
}
defaults.update(overrides)
return SimpleNamespace(**defaults)


def _make_engine_config_with_runner(runner="auto", **overrides):
"""Build a fake engine config with runner and other fields used by defaults loop."""
defaults = {
"runner": runner,
"enable_prefix_caching": True,
"block_size": 16,
"skip_tokenizer_init": True,
"enable_log_requests": True,
"disable_log_stats": True,
"kv_events_config": None,
"kv_transfer_config": None,
}
defaults.update(overrides)
return SimpleNamespace(**defaults)


class TestRunnerPreservation:
"""update_engine_config_with_dynamo must not overwrite a user-set --runner."""

def test_runner_auto_is_preserved(self):
"""When user passes --runner auto (also the vLLM default),
Dynamo must leave it alone so vLLM's own auto-detection runs."""
dynamo_cfg = _make_dynamo_config()
engine_cfg = _make_engine_config_with_runner(runner="auto")

update_engine_config_with_dynamo(dynamo_cfg, engine_cfg)

assert engine_cfg.runner == "auto"

def test_runner_pooling_preserved(self):
"""When user passes --runner pooling (for embedding models),
Dynamo must NOT overwrite it."""
dynamo_cfg = _make_dynamo_config()
engine_cfg = _make_engine_config_with_runner(runner="pooling")

update_engine_config_with_dynamo(dynamo_cfg, engine_cfg)

assert engine_cfg.runner == "pooling"

def test_runner_generate_explicit_preserved(self):
"""When user explicitly passes --runner generate, it should still be 'generate'."""
dynamo_cfg = _make_dynamo_config()
engine_cfg = _make_engine_config_with_runner(runner="generate")

update_engine_config_with_dynamo(dynamo_cfg, engine_cfg)

assert engine_cfg.runner == "generate"

def test_runner_draft_preserved(self):
"""When user passes --runner draft, Dynamo must NOT overwrite it."""
dynamo_cfg = _make_dynamo_config()
engine_cfg = _make_engine_config_with_runner(runner="draft")

update_engine_config_with_dynamo(dynamo_cfg, engine_cfg)

assert engine_cfg.runner == "draft"

def test_no_runner_attr_skipped_gracefully(self):
"""If engine_config lacks a 'runner' attr (older vLLM), no error is raised."""
dynamo_cfg = _make_dynamo_config()
engine_cfg = _make_engine_config_with_runner()
del engine_cfg.runner # simulate older vLLM without runner

update_engine_config_with_dynamo(dynamo_cfg, engine_cfg)

assert not hasattr(engine_cfg, "runner")
Loading