From d737e23f26a88b8325081042a40290d0d512b07a Mon Sep 17 00:00:00 2001 From: Tzu-Ling Date: Mon, 18 May 2026 14:34:12 -0500 Subject: [PATCH 1/5] fix(vllm): preserve user-provided --runner flag in update_engine_config_with_dynamo `update_engine_config_with_dynamo()` placed `"runner": "generate"` in the unconditional `defaults` dict, which the subsequent setattr loop applied without checking whether the user had set the value. Any `--runner pooling` (required for embedding models like Qwen3-Embedding-0.6B / google/embeddinggemma-300m) was silently overwritten and the engine crashed with: pydantic_core._pydantic_core.ValidationError: 1 validation error for ModelConfig Value error, This model does not support `--runner generate`. Move the `runner` default out of the unconditional dict and guard it: only set it to `"generate"` when `engine_config.runner == "auto"` (vLLM's default when the user did not pass `--runner`). User-provided values (`pooling`, `draft`, or explicit `generate`) are preserved. Adds `TestRunnerPreservation` with five cases: - auto -> generate default - pooling preserved (embedding model use case from #7670 / DYN-3048) - explicit generate preserved - draft preserved - missing `runner` attr (older vLLM) handled gracefully This is a rebased + black-formatted version of #7680 by @pecord-ent. GitHub issue #7670 was marked as fixed in a commit (7856cb2) that lives on a diverged branch (1 ahead, 853 behind main) and never landed, so the bug is still present on main today. Fixes #7670 Refs DYN-3048 Co-authored-by: pecord-ent Signed-off-by: Tzu-Ling --- components/src/dynamo/vllm/args.py | 12 ++- .../src/dynamo/vllm/tests/test_vllm_unit.py | 84 +++++++++++++++++++ 2 files changed, 94 insertions(+), 2 deletions(-) diff --git a/components/src/dynamo/vllm/args.py b/components/src/dynamo/vllm/args.py index fdb4cd99035d..1ac631e4af1d 100644 --- a/components/src/dynamo/vllm/args.py +++ b/components/src/dynamo/vllm/args.py @@ -238,8 +238,6 @@ def update_engine_config_with_dynamo( ensure_side_channel_host() defaults = { - # vLLM 0.13+ renamed 'task' to 'runner' - "runner": "generate", # As of vLLM >=0.10.0 the engine unconditionally calls # `sampling_params.update_from_tokenizer(...)`, so we can no longer # skip tokenizer initialisation. Setting this to **False** avoids @@ -249,6 +247,16 @@ def update_engine_config_with_dynamo( "disable_log_stats": False, } + # vLLM 0.13+ renamed 'task' to 'runner'. Only default to "generate" + # when the user did NOT explicitly set --runner (vLLM's default is "auto"). + # This preserves user-provided values like --runner pooling for embedding + # models. + if hasattr(engine_config, "runner"): + if engine_config.runner == "auto": + defaults["runner"] = "generate" + else: + logger.debug(f"Preserving user-provided runner: {engine_config.runner}") + kv_cfg = create_kv_events_config(dynamo_config, engine_config) defaults["kv_events_config"] = kv_cfg dynamo_config.use_kv_events = kv_cfg is not None and kv_cfg.enable_kv_cache_events diff --git a/components/src/dynamo/vllm/tests/test_vllm_unit.py b/components/src/dynamo/vllm/tests/test_vllm_unit.py index b3dfcf23e641..cc68a8716a7e 100644 --- a/components/src/dynamo/vllm/tests/test_vllm_unit.py +++ b/components/src/dynamo/vllm/tests/test_vllm_unit.py @@ -22,6 +22,7 @@ ensure_side_channel_host, get_host_ip, parse_args, + update_engine_config_with_dynamo, ) from dynamo.vllm.constants import DisaggregationMode from dynamo.vllm.tests.conftest import make_cli_args_fixture @@ -738,3 +739,86 @@ def test_build_sampling_params_maps_max_thinking_tokens(): } sp = build_sampling_params(request, default_sampling_params={}) assert sp.thinking_token_budget == 1024 + + +# --- update_engine_config_with_dynamo: --runner preservation tests --- + + +def _make_dynamo_config(**overrides): + """Build a minimal fake DynamoConfig for update_engine_config_with_dynamo tests.""" + defaults = { + "disaggregation_mode": DisaggregationMode.AGGREGATED, + "use_kv_events": False, + "enable_local_indexer": True, + "benchmark_mode": None, + } + defaults.update(overrides) + return SimpleNamespace(**defaults) + + +def _make_engine_config_with_runner(runner="auto", **overrides): + """Build a fake engine config with runner and other fields used by defaults loop.""" + defaults = { + "runner": runner, + "enable_prefix_caching": True, + "block_size": 16, + "skip_tokenizer_init": True, + "enable_log_requests": True, + "disable_log_stats": True, + "kv_events_config": None, + "kv_transfer_config": None, + } + defaults.update(overrides) + return SimpleNamespace(**defaults) + + +class TestRunnerPreservation: + """Tests for GitHub issue #7670: --runner must not be unconditionally overridden.""" + + def test_runner_defaults_to_generate_when_auto(self): + """When user does not pass --runner (vLLM default is 'auto'), + Dynamo should set it to 'generate'.""" + dynamo_cfg = _make_dynamo_config() + engine_cfg = _make_engine_config_with_runner(runner="auto") + + update_engine_config_with_dynamo(dynamo_cfg, engine_cfg) + + assert engine_cfg.runner == "generate" + + def test_runner_pooling_preserved(self): + """When user passes --runner pooling (for embedding models), + Dynamo must NOT overwrite it.""" + dynamo_cfg = _make_dynamo_config() + engine_cfg = _make_engine_config_with_runner(runner="pooling") + + update_engine_config_with_dynamo(dynamo_cfg, engine_cfg) + + assert engine_cfg.runner == "pooling" + + def test_runner_generate_explicit_preserved(self): + """When user explicitly passes --runner generate, it should still be 'generate'.""" + dynamo_cfg = _make_dynamo_config() + engine_cfg = _make_engine_config_with_runner(runner="generate") + + update_engine_config_with_dynamo(dynamo_cfg, engine_cfg) + + assert engine_cfg.runner == "generate" + + def test_runner_draft_preserved(self): + """When user passes --runner draft, Dynamo must NOT overwrite it.""" + dynamo_cfg = _make_dynamo_config() + engine_cfg = _make_engine_config_with_runner(runner="draft") + + update_engine_config_with_dynamo(dynamo_cfg, engine_cfg) + + assert engine_cfg.runner == "draft" + + def test_no_runner_attr_skipped_gracefully(self): + """If engine_config lacks a 'runner' attr (older vLLM), no error is raised.""" + dynamo_cfg = _make_dynamo_config() + engine_cfg = _make_engine_config_with_runner() + del engine_cfg.runner # simulate older vLLM without runner + + update_engine_config_with_dynamo(dynamo_cfg, engine_cfg) + + assert not hasattr(engine_cfg, "runner") From 9f75618dd04355561ca86f9e582d6027bf115c8f Mon Sep 17 00:00:00 2001 From: Tzu-Ling Date: Mon, 18 May 2026 14:46:51 -0500 Subject: [PATCH 2/5] fix(vllm): annotate defaults dict as Dict[str, Any] to fix mypy MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit After moving "runner" out of the literal `defaults` dict, mypy inferred the remaining literal as `dict[str, bool]` (only False values left). This broke the subsequent assignments of `kv_events_config` (Any | None), `scheduler_cls` (str), and the new conditional `runner` (str) default: args.py:256: error: Incompatible types in assignment (expression has type "str", target has type "bool") [assignment] args.py:261: error: Incompatible types in assignment (expression has type "Any | None", target has type "bool") [assignment] args.py:274: error: Incompatible types in assignment (expression has type "str", target has type "bool") [assignment] args.py:297: error: Incompatible types in assignment (expression has type "str", target has type "bool") [assignment] The dict was always heterogeneous in practice — the prior code worked only because the literal happened to include a string ("runner": "generate") which forced mypy to widen the value type. Make that explicit with an annotation rather than relying on literal inference. `Dict` and `Any` are already imported from `typing` at the top of the file. Signed-off-by: Tzu-Ling --- components/src/dynamo/vllm/args.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/src/dynamo/vllm/args.py b/components/src/dynamo/vllm/args.py index 1ac631e4af1d..e44063872620 100644 --- a/components/src/dynamo/vllm/args.py +++ b/components/src/dynamo/vllm/args.py @@ -237,7 +237,7 @@ def update_engine_config_with_dynamo( if _uses_nixl_connector(engine_config): ensure_side_channel_host() - defaults = { + defaults: Dict[str, Any] = { # As of vLLM >=0.10.0 the engine unconditionally calls # `sampling_params.update_from_tokenizer(...)`, so we can no longer # skip tokenizer initialisation. Setting this to **False** avoids From 6fad7f5b3a2e2e48108280622e00623ca5bd50c6 Mon Sep 17 00:00:00 2001 From: Tzu-Ling Date: Mon, 18 May 2026 19:10:30 -0500 Subject: [PATCH 3/5] chore(vllm): strip ticket reference from TestRunnerPreservation docstring MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Removes the "# --- update_engine_config_with_dynamo: --runner preservation tests ---" section divider and rewords the class docstring so it documents the invariant rather than the originating bug report. Per code-review feedback that ticket-tagged dividers / docstrings don't age well — the why is in the commit history. Signed-off-by: Tzu-Ling --- components/src/dynamo/vllm/tests/test_vllm_unit.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/components/src/dynamo/vllm/tests/test_vllm_unit.py b/components/src/dynamo/vllm/tests/test_vllm_unit.py index cc68a8716a7e..82c2cd43a63f 100644 --- a/components/src/dynamo/vllm/tests/test_vllm_unit.py +++ b/components/src/dynamo/vllm/tests/test_vllm_unit.py @@ -741,9 +741,6 @@ def test_build_sampling_params_maps_max_thinking_tokens(): assert sp.thinking_token_budget == 1024 -# --- update_engine_config_with_dynamo: --runner preservation tests --- - - def _make_dynamo_config(**overrides): """Build a minimal fake DynamoConfig for update_engine_config_with_dynamo tests.""" defaults = { @@ -773,7 +770,7 @@ def _make_engine_config_with_runner(runner="auto", **overrides): class TestRunnerPreservation: - """Tests for GitHub issue #7670: --runner must not be unconditionally overridden.""" + """update_engine_config_with_dynamo must not overwrite a user-set --runner.""" def test_runner_defaults_to_generate_when_auto(self): """When user does not pass --runner (vLLM default is 'auto'), From 5459ee9c61c0bf92beab8c4d1b6445f33f5b663c Mon Sep 17 00:00:00 2001 From: Tzu-Ling Date: Mon, 18 May 2026 19:34:28 -0500 Subject: [PATCH 4/5] fix(vllm): stop overriding --runner; let vLLM auto-detection run MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Review feedback on the previous version pointed out two problems with the `runner == "auto"` guard: 1. Explicit `--runner auto` (a valid user-provided value) was still silently overwritten to "generate", making vLLM's auto-detection unreachable through Dynamo. 2. The override predates vLLM having a working `--runner auto` and is no longer necessary at all — vLLM correctly detects generation, pooling, and draft runners from the model config. Remove the override entirely. vLLM's auto mode handles generation models that previously relied on this default, and pooling models that need `--runner pooling` are no longer clobbered (the original #7670 bug). Behavior change for callers who relied on the implicit `auto -> generate` substitution: none. vLLM's auto-detection resolves to the same runner for any generation model. Test updated: `test_runner_defaults_to_generate_when_auto` becomes `test_runner_auto_is_preserved`. The four other cases (pooling, explicit generate, draft, missing-attr) are unchanged. Addresses review comments from @dynamo-ops and @nnshah1 on PR #9710. Signed-off-by: Tzu-Ling --- components/src/dynamo/vllm/args.py | 15 +++++++-------- .../src/dynamo/vllm/tests/test_vllm_unit.py | 8 ++++---- 2 files changed, 11 insertions(+), 12 deletions(-) diff --git a/components/src/dynamo/vllm/args.py b/components/src/dynamo/vllm/args.py index e44063872620..1cfdb94d7913 100644 --- a/components/src/dynamo/vllm/args.py +++ b/components/src/dynamo/vllm/args.py @@ -247,15 +247,14 @@ def update_engine_config_with_dynamo( "disable_log_stats": False, } - # vLLM 0.13+ renamed 'task' to 'runner'. Only default to "generate" - # when the user did NOT explicitly set --runner (vLLM's default is "auto"). - # This preserves user-provided values like --runner pooling for embedding - # models. + # vLLM 0.13+ renamed 'task' to 'runner'. Dynamo intentionally does NOT + # override the user-supplied (or vLLM-default) runner: vLLM's "auto" mode + # detects generation vs pooling vs draft from the model config, and the + # previous unconditional `runner = "generate"` override broke embedding / + # pooling models that need `--runner pooling` and would also clobber a + # user-explicit `--runner auto`. if hasattr(engine_config, "runner"): - if engine_config.runner == "auto": - defaults["runner"] = "generate" - else: - logger.debug(f"Preserving user-provided runner: {engine_config.runner}") + logger.debug(f"Using runner={engine_config.runner} from engine args") kv_cfg = create_kv_events_config(dynamo_config, engine_config) defaults["kv_events_config"] = kv_cfg diff --git a/components/src/dynamo/vllm/tests/test_vllm_unit.py b/components/src/dynamo/vllm/tests/test_vllm_unit.py index 82c2cd43a63f..ce4b0065468a 100644 --- a/components/src/dynamo/vllm/tests/test_vllm_unit.py +++ b/components/src/dynamo/vllm/tests/test_vllm_unit.py @@ -772,15 +772,15 @@ def _make_engine_config_with_runner(runner="auto", **overrides): class TestRunnerPreservation: """update_engine_config_with_dynamo must not overwrite a user-set --runner.""" - def test_runner_defaults_to_generate_when_auto(self): - """When user does not pass --runner (vLLM default is 'auto'), - Dynamo should set it to 'generate'.""" + def test_runner_auto_is_preserved(self): + """When user passes --runner auto (also the vLLM default), + Dynamo must leave it alone so vLLM's own auto-detection runs.""" dynamo_cfg = _make_dynamo_config() engine_cfg = _make_engine_config_with_runner(runner="auto") update_engine_config_with_dynamo(dynamo_cfg, engine_cfg) - assert engine_cfg.runner == "generate" + assert engine_cfg.runner == "auto" def test_runner_pooling_preserved(self): """When user passes --runner pooling (for embedding models), From df5c99c3ea48791c456486f1771bbf1e1f6a6428 Mon Sep 17 00:00:00 2001 From: Tzu-Ling Date: Tue, 19 May 2026 11:41:42 -0500 Subject: [PATCH 5/5] chore(vllm): remove leftover comment block about --runner override MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit @nnshah1 review feedback: the comment in args.py only made sense in the context of the previous version (the one that still set a default). Now that the function doesn't touch runner at all, the comment block reads as dead context for someone arriving fresh. Move the rationale to the PR description instead. The `logger.debug(...)` line is kept — that's a runtime breadcrumb, not a code comment. Signed-off-by: Tzu-Ling --- components/src/dynamo/vllm/args.py | 6 ------ 1 file changed, 6 deletions(-) diff --git a/components/src/dynamo/vllm/args.py b/components/src/dynamo/vllm/args.py index 1cfdb94d7913..acbfd2908607 100644 --- a/components/src/dynamo/vllm/args.py +++ b/components/src/dynamo/vllm/args.py @@ -247,12 +247,6 @@ def update_engine_config_with_dynamo( "disable_log_stats": False, } - # vLLM 0.13+ renamed 'task' to 'runner'. Dynamo intentionally does NOT - # override the user-supplied (or vLLM-default) runner: vLLM's "auto" mode - # detects generation vs pooling vs draft from the model config, and the - # previous unconditional `runner = "generate"` override broke embedding / - # pooling models that need `--runner pooling` and would also clobber a - # user-explicit `--runner auto`. if hasattr(engine_config, "runner"): logger.debug(f"Using runner={engine_config.runner} from engine args")