diff --git a/components/src/dynamo/common/configuration/groups/runtime_args.py b/components/src/dynamo/common/configuration/groups/runtime_args.py index d64e4dff92eb..3212e6c84921 100644 --- a/components/src/dynamo/common/configuration/groups/runtime_args.py +++ b/components/src/dynamo/common/configuration/groups/runtime_args.py @@ -32,7 +32,6 @@ class DynamoRuntimeConfig(ConfigBase): request_plane: str event_plane: Optional[str] = None fpm_trace: bool = False - connector: list[str] enable_local_indexer: bool = True dyn_tool_call_parser: Optional[str] = None @@ -199,14 +198,6 @@ def add_arguments(self, parser: argparse.ArgumentParser) -> None: default=False, help="Persist backend forward-pass metrics to rotating gzip JSONL trace files. Also enables the backend FPM instrumentation required to produce those records.", ) - add_argument( - g, - flag_name="--connector", - env_var="DYN_CONNECTOR", - default=[], - help="[Deprecated for vLLM] Use --kv-transfer-config instead. For TRT-LLM, options: nixl, lmcache, kvbm, null, none.", - nargs="*", - ) # Optional: tool/reasoning parsers (choices from dynamo._core when available) add_argument( diff --git a/components/src/dynamo/trtllm/args.py b/components/src/dynamo/trtllm/args.py index 7cfd3c07ed79..d6940ffcc2ea 100644 --- a/components/src/dynamo/trtllm/args.py +++ b/components/src/dynamo/trtllm/args.py @@ -49,7 +49,6 @@ class Config(DynamoRuntimeConfig, DynamoTrtllmConfig): # Routing this worker set advertises in its model card; None inherits the # frontend's configuration. router_advertisement: Optional[WorkerRouterConfig] = None - connector: list[str] # Redeclare for mypy (inherited from DynamoRuntimeConfig) def validate(self) -> None: DynamoRuntimeConfig.validate(self) @@ -65,14 +64,11 @@ def validate(self) -> None: "TRT-LLM supports at most one connector entry. Use `--connector none` or `--connector kvbm`." ) elif self.connector[0] not in VALID_TRTLLM_CONNECTORS: - source = ( - f"DYN_CONNECTOR environment variable ('{os.environ['DYN_CONNECTOR']}')" - if "DYN_CONNECTOR" in os.environ - else f"shared runtime default ('{self.connector[0]}')" - ) logging.warning( - f"TRT-LLM does not support connector '{self.connector[0]}' (set via {source}). " - f"Supported connectors: {VALID_TRTLLM_CONNECTORS}. Falling back to 'none'." + "TRT-LLM does not support connector '%s'. " + "Supported connectors: %s. Falling back to 'none'.", + self.connector[0], + VALID_TRTLLM_CONNECTORS, ) self.connector = ["none"] diff --git a/components/src/dynamo/trtllm/backend_args.py b/components/src/dynamo/trtllm/backend_args.py index aba33f7eeb98..2caf8ebe2a60 100644 --- a/components/src/dynamo/trtllm/backend_args.py +++ b/components/src/dynamo/trtllm/backend_args.py @@ -105,6 +105,15 @@ def add_arguments(self, parser: argparse.ArgumentParser) -> None: "Dynamo router's selected rank and requires a TensorRT-LLM build containing " "NVIDIA/TensorRT-LLM#16815 or equivalent.", ) + add_argument( + g, + flag_name="--connector", + env_var="DYN_CONNECTOR", + default=[], + help="KV cache connector for the TensorRT-LLM engine. Accepts at most " + "one value: 'kvbm' enables the KVBM integration, 'none' disables it.", + nargs="*", + ) add_argument( g, flag_name="--kv-block-size", @@ -490,6 +499,7 @@ class DynamoTrtllmConfig(ConfigBase): enable_attention_dp: bool conversation_affinity: bool conversation_affinity_dp_rank_source: str + connector: list[str] kv_block_size: int gpus_per_node: Optional[int] = None max_batch_size: int diff --git a/components/src/dynamo/vllm/args.py b/components/src/dynamo/vllm/args.py index 896b13e2db6b..5442e61e30b9 100644 --- a/components/src/dynamo/vllm/args.py +++ b/components/src/dynamo/vllm/args.py @@ -3,7 +3,6 @@ import argparse import ipaddress -import json import logging import os import socket @@ -224,9 +223,6 @@ def update_dynamo_config_with_engine( "Please ensure the file exists and the path is correct." ) - # --connector is no longer supported for vLLM. Raise hard error if explicitly set. - _reject_connector_flag(dynamo_config) - # If disaggregation mode is prefill, require explicit --kv-transfer-config has_kv_transfer_config = ( hasattr(engine_config, "kv_transfer_config") @@ -237,16 +233,12 @@ def update_dynamo_config_with_engine( and not has_kv_transfer_config ): raise ValueError( - "--connector is deprecated and the default is no longer nixl. " "When using --disaggregation-mode prefill, you must explicitly " "provide --kv-transfer-config. Example:\n" " --kv-transfer-config " '\'{"kv_connector":"NixlConnector","kv_role":"kv_both"}\'' ) - # Clear connector list (no longer used for vLLM) - dynamo_config.connector = [] # type: ignore[assignment] - def _unsupported_fpm_trace_role(dynamo_config: Config) -> Optional[str]: """Return the worker role when trace-based FPM activation is unsupported.""" @@ -490,98 +482,6 @@ def _uses_dynamo_connector(engine_config: AsyncEngineArgs) -> bool: return False -def _connector_to_kv_transfer_json(connectors: list[str]) -> str: - """Convert a legacy --connector list to the equivalent --kv-transfer-config JSON. - - Used in error messages to help users migrate. - """ - multi_connectors = [] - for conn in connectors: - c = conn.lower() - if c == "lmcache": - multi_connectors.append( - {"kv_connector": "LMCacheConnectorV1", "kv_role": "kv_both"} - ) - elif c == "flexkv": - multi_connectors.append( - {"kv_connector": "FlexKVConnectorV1", "kv_role": "kv_both"} - ) - elif c == "nixl": - multi_connectors.append( - {"kv_connector": "NixlConnector", "kv_role": "kv_both"} - ) - elif c == "kvbm": - multi_connectors.append( - { - "kv_connector": "DynamoConnector", - "kv_connector_module_path": "kvbm.vllm_integration.connector", - "kv_role": "kv_both", - } - ) - - if len(multi_connectors) == 1: - return json.dumps(multi_connectors[0]) - - return json.dumps( - { - "kv_connector": "PdConnector", - "kv_role": "kv_both", - "kv_connector_extra_config": {"connectors": multi_connectors}, - "kv_connector_module_path": "kvbm.vllm_integration.connector", - } - ) - - -def _reject_connector_flag(dynamo_config: Config) -> None: - """Raise ValueError if --connector was explicitly set (CLI or DYN_CONNECTOR env var). - - The --connector flag is no longer supported for the vLLM backend. - Users must use --kv-transfer-config instead. - """ - connector_list = dynamo_config.connector or [] - - # Check if --connector was explicitly provided via CLI or DYN_CONNECTOR env var - env_connector = os.environ.get("DYN_CONNECTOR") - explicitly_set = bool(connector_list) or (env_connector is not None) - - if not explicitly_set: - return - - # Normalize: "none"/"null" means no connector - normalized = [c.lower() for c in connector_list] - if normalized and all(c in ("none", "null") for c in normalized): - # --connector none/null: tell user it's no longer needed - raise ValueError( - "--connector is no longer supported for the vLLM backend. " - "'--connector none' is no longer needed — the default is already " - "no connector. Simply remove the --connector flag." - ) - - # Active connectors: show migration path - if normalized: - equiv = _connector_to_kv_transfer_json(normalized) - raise ValueError( - "--connector is no longer supported for the vLLM backend. " - "Use --kv-transfer-config instead.\n" - f" Equivalent: --kv-transfer-config '{equiv}'" - ) - - # DYN_CONNECTOR env var set but parsed to empty list - if env_connector is not None: - env_values = [v.strip().lower() for v in env_connector.split() if v.strip()] - if env_values and not all(v in ("none", "null") for v in env_values): - equiv = _connector_to_kv_transfer_json(env_values) - raise ValueError( - "The DYN_CONNECTOR environment variable is no longer supported " - "for the vLLM backend. Use --kv-transfer-config instead.\n" - f" Equivalent: --kv-transfer-config '{equiv}'" - ) - raise ValueError( - "The DYN_CONNECTOR environment variable is no longer supported " - "for the vLLM backend. Use --kv-transfer-config instead." - ) - - def get_host_ip() -> str: """Get a routable IP address of the host for NIXL side-channel coordination. diff --git a/components/src/dynamo/vllm/tests/omni/test_omni_args.py b/components/src/dynamo/vllm/tests/omni/test_omni_args.py index c114cf3cd63a..900335c976ae 100644 --- a/components/src/dynamo/vllm/tests/omni/test_omni_args.py +++ b/components/src/dynamo/vllm/tests/omni/test_omni_args.py @@ -55,7 +55,6 @@ def _make_omni_config(**overrides) -> OmniConfig: "discovery_backend": "etcd", "request_plane": "tcp", "event_plane": "nats", - "connector": [], "enable_local_indexer": True, "dyn_tool_call_parser": None, "dyn_reasoning_parser": None, diff --git a/components/src/dynamo/vllm/tests/test_vllm_unit.py b/components/src/dynamo/vllm/tests/test_vllm_unit.py index 54766f5a1eb4..c6f64113e37e 100644 --- a/components/src/dynamo/vllm/tests/test_vllm_unit.py +++ b/components/src/dynamo/vllm/tests/test_vllm_unit.py @@ -20,7 +20,6 @@ import dynamo.llm as dynamo_llm from dynamo.vllm import envs from dynamo.vllm.args import ( - _connector_to_kv_transfer_json, _is_routable, _uses_dynamo_connector, _uses_nixl_connector, @@ -234,31 +233,6 @@ def test_removed_multimodal_role_flags_are_rejected(flag, mock_vllm_cli): parse_args() -# --connector removal tests - - -def test_connector_nixl_raises_error_with_migration_hint(mock_vllm_cli): - """Test that --connector nixl raises ValueError with --kv-transfer-config hint.""" - mock_vllm_cli("--model", "Qwen/Qwen3-0.6B", "--connector", "nixl") - with pytest.raises(ValueError, match="--connector is no longer supported"): - parse_args() - - -def test_connector_none_raises_error(mock_vllm_cli): - """Test that --connector none raises ValueError telling user it's no longer needed.""" - mock_vllm_cli("--model", "Qwen/Qwen3-0.6B", "--connector", "none") - with pytest.raises(ValueError, match="no longer needed"): - parse_args() - - -def test_env_var_dyn_connector_raises_error(monkeypatch, mock_vllm_cli): - """Test that DYN_CONNECTOR env var raises error for vLLM backend.""" - monkeypatch.setenv("DYN_CONNECTOR", "nixl") - mock_vllm_cli("--model", "Qwen/Qwen3-0.6B") - with pytest.raises(ValueError, match="no longer supported"): - parse_args() - - def test_model_express_url_is_accepted_for_compatibility(mock_vllm_cli): """Test that legacy ModelExpress manifests still parse.""" mock_vllm_cli( @@ -292,22 +266,6 @@ def test_prefill_worker_without_kv_transfer_config_raises(mock_vllm_cli): parse_args() -def test_connector_to_kv_transfer_json_single(): - """Test _connector_to_kv_transfer_json returns valid JSON for a single connector.""" - result = json.loads(_connector_to_kv_transfer_json(["nixl"])) - assert result == {"kv_connector": "NixlConnector", "kv_role": "kv_both"} - - -def test_connector_to_kv_transfer_json_multi(): - """Test _connector_to_kv_transfer_json wraps multiple connectors in PdConnector.""" - result = json.loads(_connector_to_kv_transfer_json(["kvbm", "nixl"])) - assert result["kv_connector"] == "PdConnector" - nested = result["kv_connector_extra_config"]["connectors"] - nested_names = [c["kv_connector"] for c in nested] - assert "DynamoConnector" in nested_names - assert "NixlConnector" in nested_names - - # _uses_nixl_connector / _uses_dynamo_connector tests diff --git a/docs/fern/pages/reference/backends/tensorrt-llm-configuration.mdx b/docs/fern/pages/reference/backends/tensorrt-llm-configuration.mdx index c47efa964323..6632281badce 100644 --- a/docs/fern/pages/reference/backends/tensorrt-llm-configuration.mdx +++ b/docs/fern/pages/reference/backends/tensorrt-llm-configuration.mdx @@ -160,6 +160,14 @@ Each field is both a CLI flag and an environment variable. The CLI flag takes pr Environment variable: `DYN_TRTLLM_PUBLISH_KV_EVENTS` + + KV cache connector for the TensorRT-LLM engine. Accepts at most one value: `kvbm` enables the [KVBM](../../developer-guide/knowledge-base/modular-components/kvbm/kvbm-guide.md) integration, `none` disables it. When unset, no connector is configured. This flag is specific to TensorRT-LLM; other backends configure KV transfer through their own engine options. + + Allowed values: kvbm none + + Environment variable: `DYN_CONNECTOR` + + ## Engine args passthrough diff --git a/docs/fern/pages/reference/backends/vllm-configuration.mdx b/docs/fern/pages/reference/backends/vllm-configuration.mdx index 47e80f754c70..baf043683308 100644 --- a/docs/fern/pages/reference/backends/vllm-configuration.mdx +++ b/docs/fern/pages/reference/backends/vllm-configuration.mdx @@ -91,12 +91,6 @@ python -m dynamo.vllm \ Only the prefill worker is required to set it, but both halves of a NIXL pair must agree on a connector for transfers to succeed. Pass the same `--kv-transfer-config` value to the decode worker, as the [disaggregated vLLM launch script](https://github.com/ai-dynamo/dynamo/blob/main/examples/backends/vllm/launch/disagg.sh) does. -The earlier `--connector` flag is no longer accepted by the vLLM backend. Setting it — on the command line or through the `DYN_CONNECTOR` environment variable — raises a `ValueError` during argument parsing. The message depends on the value: - -- An active connector, such as `--connector nixl` or `DYN_CONNECTOR=nixl`, reports the equivalent `--kv-transfer-config` JSON to use instead. -- `--connector none` or `--connector null` reports that the flag is no longer needed, because no connector is already the default. There is no equivalent value to migrate to, so none is shown. -- `DYN_CONNECTOR` set to an empty or whitespace-only value reports that the variable is no longer supported, without an equivalent value. - ## Worker role and disaggregation These flags control which role this worker plays in a disaggregated deployment. The default when no `--disaggregation-mode` is set is aggregated (`agg`). diff --git a/docs/fern/pages/reference/components/runtime-configuration.mdx b/docs/fern/pages/reference/components/runtime-configuration.mdx index 50502a380ddd..3d175f92dada 100644 --- a/docs/fern/pages/reference/components/runtime-configuration.mdx +++ b/docs/fern/pages/reference/components/runtime-configuration.mdx @@ -132,12 +132,6 @@ Unless a field is marked environment-only, it has both a CLI flag and an environ Environment variable: `DYN_EVENT_PLANE` - - KV-cache transfer connector. Accepts zero or more values. Deprecated for vLLM — use `--kv-transfer-config` instead. For TRT-LLM, valid options are `nixl`, `lmcache`, `kvbm`, `null`, and `none`. - - Environment variable: `DYN_CONNECTOR` - - ## Parsing diff --git a/recipes/nemotron-3-super-fp8/README.md b/recipes/nemotron-3-super-fp8/README.md index 7d18e3eaac66..2fba717ba47e 100644 --- a/recipes/nemotron-3-super-fp8/README.md +++ b/recipes/nemotron-3-super-fp8/README.md @@ -140,10 +140,10 @@ These recipes target Dynamo v1.0.0. To run on v0.9.1 containers, the following c ### vLLM (`vllm-runtime:0.9.1`) - Change image tags from `:1.1.1` to `:0.9.1` -- **Add** `--connector none` to worker args (required in 0.9.1 to disable nixl KV connector; rejected in 1.0) - Change `--dyn-reasoning-parser` from `nemotron_nano` to `deepseek_r1` (nemotron_nano reasoning parser is broken in 0.9.1) - `enable_thinking: false` will **not work** with `deepseek_r1` parser (response content goes to `reasoning_content`, `content` is null) - `--mamba-cache-mode align` is still needed (0.9.1 ships vLLM 0.14.1, also affected by [vllm#34865](https://github.com/vllm-project/vllm/issues/34865)) +- **Add** `--connector none` to worker args (required in 0.9.1 to disable nixl KV connector; rejected in 1.0) ### TensorRT-LLM (`tensorrtllm-runtime:0.9.1`) - Change image tags from `:1.1.1` to `:0.9.1`