diff --git a/components/src/dynamo/common/configuration/groups/runtime_args.py b/components/src/dynamo/common/configuration/groups/runtime_args.py index 8a64cbcb7c58..417974fb90dd 100644 --- a/components/src/dynamo/common/configuration/groups/runtime_args.py +++ b/components/src/dynamo/common/configuration/groups/runtime_args.py @@ -266,7 +266,10 @@ def add_arguments(self, parser: argparse.ArgumentParser) -> None: env_var="DYN_ENDPOINT_TYPES", default="chat,completions", obsolete_flag="--dyn-endpoint-types", - help="Comma-separated list of endpoint types to enable. Options: 'chat', 'completions'. Use 'completions' for models without chat templates.", + help="Comma-separated list of endpoint types to enable. Options: " + "'chat', 'completions', or 'none'. Use 'completions' for models " + "without chat templates. Use 'none' for topology-only workers " + "fronted by another Dynamo service.", ) add_argument( diff --git a/components/src/dynamo/common/utils/endpoint_types.py b/components/src/dynamo/common/utils/endpoint_types.py index e66bc81d0dbb..7dc5c2191799 100644 --- a/components/src/dynamo/common/utils/endpoint_types.py +++ b/components/src/dynamo/common/utils/endpoint_types.py @@ -11,14 +11,15 @@ def parse_endpoint_types(endpoint_types_str: str) -> ModelType: Args: endpoint_types_str: Comma-separated list of endpoint types. - Valid values: 'chat', 'completions' - Examples: 'chat', 'completions', 'chat,completions' + Valid values: 'chat', 'completions', or 'none' + Examples: 'chat', 'completions', 'chat,completions', 'none' Returns: ModelType flags combined with bitwise OR Raises: - ValueError: If any invalid endpoint type is provided or string is empty + ValueError: If any invalid endpoint type is provided, 'none' is mixed + with other endpoint types, or the string is empty Examples: >>> parse_endpoint_types("chat") @@ -27,6 +28,8 @@ def parse_endpoint_types(endpoint_types_str: str) -> ModelType: ModelType.Completions >>> parse_endpoint_types("chat,completions") ModelType.Chat | ModelType.Completions + >>> parse_endpoint_types("none") + ModelType.Empty """ if not endpoint_types_str or not endpoint_types_str.strip(): raise ValueError("Endpoint types string cannot be empty") @@ -36,6 +39,13 @@ def parse_endpoint_types(endpoint_types_str: str) -> ModelType: if not types: raise ValueError("No valid endpoint types provided") + if "none" in types: + if len(types) > 1: + raise ValueError( + "Endpoint type 'none' cannot be combined with other endpoint types" + ) + return ModelType.Empty + result: ModelType | None = None for t in types: if t == "chat": @@ -44,7 +54,8 @@ def parse_endpoint_types(endpoint_types_str: str) -> ModelType: flag = ModelType.Completions else: raise ValueError( - f"Invalid endpoint type: '{t}'. Valid options: 'chat', 'completions'" + f"Invalid endpoint type: '{t}'. Valid options: " + "'chat', 'completions', 'none'" ) result = flag if result is None else result | flag diff --git a/components/src/dynamo/common/utils/tests/test_endpoint_types.py b/components/src/dynamo/common/utils/tests/test_endpoint_types.py new file mode 100644 index 000000000000..0cdd4aeb2529 --- /dev/null +++ b/components/src/dynamo/common/utils/tests/test_endpoint_types.py @@ -0,0 +1,36 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Unit tests for endpoint type parsing.""" + +import pytest + +from dynamo.common.utils.endpoint_types import parse_endpoint_types +from dynamo.llm import ModelType + +pytestmark = [ + pytest.mark.unit, + pytest.mark.gpu_0, + pytest.mark.pre_merge, +] + + +def test_parse_endpoint_types_allows_topology_only_workers(): + assert parse_endpoint_types("none") == ModelType.Empty + + +@pytest.mark.parametrize("value", ["chat,none", "none,completions"]) +def test_parse_endpoint_types_rejects_mixing_none_with_public_surfaces(value): + with pytest.raises(ValueError, match="'none' cannot be combined"): + parse_endpoint_types(value) + + +@pytest.mark.parametrize("value", ["", " ", ",,,"]) +def test_parse_endpoint_types_rejects_empty_input(value): + with pytest.raises(ValueError): + parse_endpoint_types(value) + + +def test_parse_endpoint_types_invalid_option_lists_none(): + with pytest.raises(ValueError, match="'chat', 'completions', 'none'"): + parse_endpoint_types("responses") diff --git a/components/src/dynamo/thunderagent_router/README.md b/components/src/dynamo/thunderagent_router/README.md index 2bd6e066f508..75d6e4a1c1d1 100644 --- a/components/src/dynamo/thunderagent_router/README.md +++ b/components/src/dynamo/thunderagent_router/README.md @@ -36,6 +36,7 @@ uv pip install -e . # 1. Start your Dynamo workers (vLLM example, with KV events on) python -m dynamo.vllm \ --model --tensor-parallel-size \ + --endpoint-types none \ --kv-events-config '{"publisher":"zmq","topic":"kv-events", "endpoint":"tcp://*:20080", "enable_kv_cache_events":true}' @@ -51,6 +52,12 @@ python -m dynamo.thunderagent_router \ python -m dynamo.frontend --router-mode round-robin ``` +Use `--endpoint-types none` on workers wrapped by ThunderAgent so they register +only for topology and readiness. ThunderAgent registers the public +chat/completions surface; if the wrapped backend also advertises that surface +for the same model, the frontend may route requests directly to the backend and +bypass ThunderAgent lifecycle handling such as `x-dynamo-session-final`. + The control-loop knobs (`--pause-threshold`, `--pause-target`, `--resume-hysteresis`, `--scheduler-interval-seconds`, …) and their defaults are documented in [docs/agents/thunderagent-router.md](../../../../docs/fern/agents/thunderagent-router.md#utilization-driven-control-loop).