Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -266,7 +266,10 @@ def add_arguments(self, parser: argparse.ArgumentParser) -> None:
env_var="DYN_ENDPOINT_TYPES",
default="chat,completions",
obsolete_flag="--dyn-endpoint-types",
help="Comma-separated list of endpoint types to enable. Options: 'chat', 'completions'. Use 'completions' for models without chat templates.",
help="Comma-separated list of endpoint types to enable. Options: "
"'chat', 'completions', or 'none'. Use 'completions' for models "
"without chat templates. Use 'none' for topology-only workers "
"fronted by another Dynamo service.",
Comment thread
cpakkamisaac-sae marked this conversation as resolved.
)

add_argument(
Expand Down
19 changes: 15 additions & 4 deletions components/src/dynamo/common/utils/endpoint_types.py
Original file line number Diff line number Diff line change
Expand Up @@ -11,14 +11,15 @@ def parse_endpoint_types(endpoint_types_str: str) -> ModelType:

Args:
endpoint_types_str: Comma-separated list of endpoint types.
Valid values: 'chat', 'completions'
Examples: 'chat', 'completions', 'chat,completions'
Valid values: 'chat', 'completions', or 'none'
Examples: 'chat', 'completions', 'chat,completions', 'none'

Returns:
ModelType flags combined with bitwise OR

Raises:
ValueError: If any invalid endpoint type is provided or string is empty
ValueError: If any invalid endpoint type is provided, 'none' is mixed
with other endpoint types, or the string is empty

Examples:
>>> parse_endpoint_types("chat")
Expand All @@ -27,6 +28,8 @@ def parse_endpoint_types(endpoint_types_str: str) -> ModelType:
ModelType.Completions
>>> parse_endpoint_types("chat,completions")
ModelType.Chat | ModelType.Completions
>>> parse_endpoint_types("none")
ModelType.Empty
"""
if not endpoint_types_str or not endpoint_types_str.strip():
raise ValueError("Endpoint types string cannot be empty")
Expand All @@ -36,6 +39,13 @@ def parse_endpoint_types(endpoint_types_str: str) -> ModelType:
if not types:
raise ValueError("No valid endpoint types provided")

if "none" in types:
if len(types) > 1:
raise ValueError(
"Endpoint type 'none' cannot be combined with other endpoint types"
)
return ModelType.Empty

result: ModelType | None = None
for t in types:
if t == "chat":
Expand All @@ -44,7 +54,8 @@ def parse_endpoint_types(endpoint_types_str: str) -> ModelType:
flag = ModelType.Completions
else:
raise ValueError(
f"Invalid endpoint type: '{t}'. Valid options: 'chat', 'completions'"
f"Invalid endpoint type: '{t}'. Valid options: "
"'chat', 'completions', 'none'"
)

result = flag if result is None else result | flag
Expand Down
36 changes: 36 additions & 0 deletions components/src/dynamo/common/utils/tests/test_endpoint_types.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,36 @@
# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
# SPDX-License-Identifier: Apache-2.0

"""Unit tests for endpoint type parsing."""

import pytest

from dynamo.common.utils.endpoint_types import parse_endpoint_types
from dynamo.llm import ModelType

pytestmark = [
pytest.mark.unit,
pytest.mark.gpu_0,
pytest.mark.pre_merge,
]


def test_parse_endpoint_types_allows_topology_only_workers():
assert parse_endpoint_types("none") == ModelType.Empty


@pytest.mark.parametrize("value", ["chat,none", "none,completions"])
def test_parse_endpoint_types_rejects_mixing_none_with_public_surfaces(value):
with pytest.raises(ValueError, match="'none' cannot be combined"):
parse_endpoint_types(value)


@pytest.mark.parametrize("value", ["", " ", ",,,"])
def test_parse_endpoint_types_rejects_empty_input(value):
with pytest.raises(ValueError):
parse_endpoint_types(value)


def test_parse_endpoint_types_invalid_option_lists_none():
with pytest.raises(ValueError, match="'chat', 'completions', 'none'"):
parse_endpoint_types("responses")
7 changes: 7 additions & 0 deletions components/src/dynamo/thunderagent_router/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,7 @@ uv pip install -e .
# 1. Start your Dynamo workers (vLLM example, with KV events on)
python -m dynamo.vllm \
--model <model> --tensor-parallel-size <N> \
--endpoint-types none \
--kv-events-config '{"publisher":"zmq","topic":"kv-events",
"endpoint":"tcp://*:20080",
"enable_kv_cache_events":true}'
Expand All @@ -51,6 +52,12 @@ python -m dynamo.thunderagent_router \
python -m dynamo.frontend --router-mode round-robin
```

Use `--endpoint-types none` on workers wrapped by ThunderAgent so they register
only for topology and readiness. ThunderAgent registers the public
chat/completions surface; if the wrapped backend also advertises that surface
for the same model, the frontend may route requests directly to the backend and
bypass ThunderAgent lifecycle handling such as `x-dynamo-session-final`.

The control-loop knobs (`--pause-threshold`, `--pause-target`,
`--resume-hysteresis`, `--scheduler-interval-seconds`, …) and their defaults are
documented in [docs/agents/thunderagent-router.md](../../../../docs/fern/agents/thunderagent-router.md#utilization-driven-control-loop).
Expand Down
Loading