Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions Cargo.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

19 changes: 17 additions & 2 deletions components/src/dynamo/sglang/publisher.py
Original file line number Diff line number Diff line change
Expand Up @@ -138,6 +138,21 @@ def format_zmq_endpoint(endpoint_template: str, ip_address: str) -> str:
return NetworkAddress(ip_address, parsed.port).to_tcp()


_WILDCARD_HOSTS = {"*", "0.0.0.0", "::"}


def kv_event_connect_ip(endpoint: str) -> str:
"""Return the address the local subscriber uses to reach SGLang's KV event publisher.

SGLang binds a wildcard endpoint without ZMQ_IPV6, so it listens on IPv4 only.
Loopback reaches that listener on every host, including IPv6-only hosts where the
host's own address is IPv6.
"""
if urlparse(endpoint).hostname in _WILDCARD_HOSTS:
return "127.0.0.1"
return get_local_ip_auto()


# Note: We use SGLang's ZmqEventPublisher.offset_endpoint_port() directly
# to ensure perfect alignment between publisher (SGLang) and subscriber (dynamo).
# This is the same pattern used by dynamo+vLLM.
Expand Down Expand Up @@ -322,7 +337,7 @@ def init_kv_event_publish(self) -> List[KvEventPublisher]:
raise ValueError(
"sglang kv_events_config is set but missing 'endpoint'"
)
local_ip = get_local_ip_auto()
connect_ip = kv_event_connect_ip(base_ep)

# Determine DP attention configuration
dp_ranks = get_local_dp_rank_range(self.server_args)
Expand All @@ -344,7 +359,7 @@ def init_kv_event_publish(self) -> List[KvEventPublisher]:
)
continue

zmq_ep = format_zmq_endpoint(zmq_ep, local_ip)
zmq_ep = format_zmq_endpoint(zmq_ep, connect_ip)

logging.info(
f"Setting up ZMQ kv event subscriber for dp_rank={dp_rank} "
Expand Down
17 changes: 17 additions & 0 deletions components/src/dynamo/sglang/tests/test_sglang_publisher.py
Original file line number Diff line number Diff line change
Expand Up @@ -441,6 +441,23 @@ def cleanup(self):
assert metrics_task.cancelled()


@pytest.mark.parametrize(
("endpoint", "expected"),
[
("tcp://*:5557", "127.0.0.1"),
("tcp://0.0.0.0:5557", "127.0.0.1"),
("tcp://[::]:5557", "127.0.0.1"),
("tcp://192.0.2.10:5557", "2001:db8::7"),
],
)
def test_kv_event_connect_ip_uses_loopback_for_wildcard_binds(
monkeypatch, endpoint, expected
):
monkeypatch.setattr(publisher_mod, "get_local_ip_auto", lambda: "2001:db8::7")

assert publisher_mod.kv_event_connect_ip(endpoint) == expected


def test_init_kv_event_publish_uses_worker_id_override(monkeypatch):
calls = []

Expand Down
6 changes: 3 additions & 3 deletions deploy/helm/charts/platform/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@ limitations under the License.

A Helm chart for NVIDIA Dynamo Platform.

![Version: 1.5.0](https://img.shields.io/badge/Version-1.5.0-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square)
![Version: 1.5.1](https://img.shields.io/badge/Version-1.5.1-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square)

## 🚀 Overview

Expand Down Expand Up @@ -146,7 +146,7 @@ Kubernetes: `>=1.30.0-0`

| Repository | Name | Version |
|------------|------|---------|
| file://components/operator | dynamo-operator | 1.5.0 |
| file://components/operator | dynamo-operator | 1.5.1 |
| https://charts.bitnami.com/bitnami | etcd | 12.0.18 |
| https://nats-io.github.io/k8s/helm/charts/ | nats | 1.3.2 |
| oci://ghcr.io/ai-dynamo/grove | grove(grove-charts) | v0.1.0-alpha.13 |
Expand Down Expand Up @@ -183,7 +183,7 @@ Kubernetes: `>=1.30.0-0`
| dynamo-operator.controllerManager.leaderElection.id | string | `""` | Leader election ID for cluster-wide coordination. WARNING: All cluster-wide operators must use the SAME ID to prevent split-brain. Different IDs would allow multiple leaders simultaneously. |
| dynamo-operator.controllerManager.leaderElection.namespace | string | `""` | Namespace for leader election leases (only used in cluster-wide mode). If empty, defaults to kube-system for cluster-wide coordination. All cluster-wide operators should use the SAME namespace for proper leader election. |
| dynamo-operator.controllerManager.manager.image.repository | string | `"nvcr.io/nvidia/ai-dynamo/kubernetes-operator"` | Official NVIDIA Dynamo operator image repository |
| dynamo-operator.controllerManager.manager.image.tag | string | `"1.5.0"` | Image tag (leave empty to use chart default) |
| dynamo-operator.controllerManager.manager.image.tag | string | `"1.5.1"` | Image tag (leave empty to use chart default) |
| dynamo-operator.controllerManager.manager.image.pullPolicy | string | `"IfNotPresent"` | Image pull policy - when to pull the image |
| dynamo-operator.controllerManager.manager.args[0] | string | `"--health-probe-bind-address=:8081"` | Health probe endpoint for Kubernetes health checks |
| dynamo-operator.controllerManager.manager.args[1] | string | `"--metrics-bind-address=127.0.0.1:8080"` | Metrics endpoint for Prometheus scraping (localhost only for security) |
Expand Down
9 changes: 8 additions & 1 deletion deploy/operator/internal/dynamo/backend_vllm.go
Original file line number Diff line number Diff line change
Expand Up @@ -168,9 +168,16 @@ def _k8s_api():
ns = open(f"{SA}/namespace").read()
return f"https://kubernetes.default.svc/api/v1/namespaces/{ns}/pods"

def resolve_leader_ip():
# gethostbyname is IPv4-only; fall back to IPv6 for IPv6-only clusters.
try:
return socket.gethostbyname(host)
except socket.gaierror:
return socket.getaddrinfo(host, None, socket.AF_INET6, socket.SOCK_STREAM)[0][4][0]

def leader_pod_is_healthy():
try:
ip = socket.gethostbyname(host)
ip = resolve_leader_ip()
except socket.gaierror:
return False, "DNS resolution failed", None, None
try:
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,8 @@ Every CLI argument has a corresponding environment variable. The CLI argument ta
## HTTP and networking

<ParamField path="--http-host" type="string" default="0.0.0.0">
HTTP listen address.
HTTP listen address: an IPv4 or IPv6 address, with optional brackets around IPv6. `0.0.0.0`
accepts IPv4 connections only; on IPv6-only hosts, set `::`.

Environment variable: `DYN_HTTP_HOST`
</ParamField>
Expand Down
90 changes: 90 additions & 0 deletions docs/fern/pages/reference/components/runtime-configuration.mdx
Original file line number Diff line number Diff line change
Expand Up @@ -123,6 +123,56 @@ Unless a field is marked environment-only, it has both a CLI flag and an environ
<span className="enum-values"><span className="enum-label">Allowed values:</span> <Badge intent="note" minimal>json</Badge> <Badge intent="note" minimal>msgpack</Badge></span>
</ParamField>

<ParamField path="DYN_TCP_RPC_HOST" type="string" default="auto-detected local address">
Host for the TCP request-plane server. Accepts IPv4 and IPv6 literals, bracketed IPv6 literals,
wildcards, and interface names, including aliases such as `eth0:1`. If unset or empty, Dynamo
selects the first usable non-loopback IPv4 address, then IPv6, then IPv4 loopback, then IPv6
loopback. On Unix, automatic selection excludes interfaces that are down. Dynamo trims surrounding
whitespace from configured hosts; a value containing only whitespace still fails server startup.

For a named interface, Dynamo uses the same address selection order, restricted to that interface.
It selects the first usable address in each category in operating-system enumeration order. An
interface with only loopback addresses uses IPv4 loopback, then IPv6 loopback. Explicit interface
lookup does not filter by interface state.

A wildcard uses the same automatic interface inventory to select a usable non-loopback address in
its requested family for advertisement. If only the other family has one, Dynamo switches the bind
wildcard to that family. If neither family has one, Dynamo prefers loopback in the requested family
before trying the other family. If no
loopback address is found, it uses the requested family's standard loopback address. The bind
wildcard uses the selected address family. Wildcards are never advertised. IPv4 link-local
addresses are accepted. Multicast, broadcast, and IPv6 link-local addresses are rejected.

IPv4-mapped IPv6 addresses use IPv4 rules. For example, `::ffff:0.0.0.0` uses the `0.0.0.0`
wildcard rules and advertises a concrete address.

The server resolves the address once at startup. A loopback fallback persists until restart, and
an interface-enumeration error fails server startup.
</ParamField>

<ParamField path="DYN_TCP_RPC_PORT" type="integer" default="0">
Port used by the TCP request-plane server. Accepted values are integers from `0` to `65535`.
When the value is unset, empty, or `0`, the operating system assigns a free port. Dynamo does not
trim whitespace. An invalid value, including a value with surrounding whitespace, silently falls
back to an operating-system-assigned port.
</ParamField>

<ParamField path="DYN_TCP_RESPONSE_STREAM_HOST" type="string" default="auto-detected local address">
Host for the TCP response-stream server. It accepts the same address syntax and uses the same
address selection and wildcard-family rules as `DYN_TCP_RPC_HOST`. Dynamo trims surrounding
whitespace; an empty value is unset, but a whitespace-only value fails startup. With a wildcard
host, the TCP listener binds the selected wildcard.

The server resolves the address once at startup. A loopback fallback persists until restart, and
an interface-enumeration error fails server startup.
</ParamField>

<ParamField path="DYN_TCP_RESPONSE_STREAM_PORT" type="integer" default="0">
Port used by the TCP response stream server. When unset or `0`, the operating system assigns a
free port. Values must be integers from `0` to `65535`. Empty values and surrounding whitespace
fail server startup.
</ParamField>

<ParamField path="--event-plane" type="string" default="zmq">
Event publishing transport. ZMQ is the default for every discovery backend; select `nats`
explicitly to use NATS Core. See [Event Plane](../../developer-guide/knowledge-base/concepts/system-architecture/architecture.md#event-plane) for design details.
Expand All @@ -132,6 +182,37 @@ Unless a field is marked environment-only, it has both a CLI flag and an environ
Environment variable: `DYN_EVENT_PLANE`
</ParamField>

<ParamField path="DYN_EVENT_PLANE_HOST" type="string" default="auto-detected local address">
IPv4 or IPv6 address, or exact network interface name, advertised by direct ZMQ event publishers.
For an interface name, Dynamo selects the first usable IPv4 address, then IPv6. Dynamo trims
whitespace and treats an empty value as unset. Unspecified addresses such as `0.0.0.0` and `::` are
invalid, including IPv4-mapped wildcards such as `::ffff:0.0.0.0`. When unset, Dynamo selects a
usable non-loopback IPv4 address first, then IPv6. On Unix, automatic selection excludes down
interfaces. If neither family has a usable non-loopback address, Dynamo selects IPv4 loopback,
then IPv6 loopback, and falls back to `127.0.0.1` if neither is found. Remote subscribers cannot
reach a loopback address. Automatic selection can choose an IPv4 link-local address or a bridge
address. On hosts with several interfaces, set this variable to a reachable IP address or interface
if automatic selection chooses the wrong address.

Direct ZMQ publishers bind `0.0.0.0` for an advertised IPv4 address or `[::]` for an advertised
IPv6 address. Discovery publishes the concrete address with the port assigned to that listener.
An explicit override must identify an address reachable by subscribers. The IPv6 wildcard can
also accept IPv4 connections; keep ZMQ listeners on trusted networks.

Event-plane PUB/SUB sockets support IPv4 addresses and bracketed IPv6 addresses, including
connections to configured ZMQ brokers. Hostnames retain IPv4 resolution. For an IPv6 broker,
use a bracketed address such as `tcp://[2001:db8::10]:5555`; Dynamo rejects an unbracketed IPv6
address. This variable does not affect NATS
or ZMQ broker mode.
</ParamField>

<Note>
On IPv6-only hosts, set `DYN_HTTP_HOST=::` on the Frontend and `DYN_SYSTEM_HOST=::` on workers. Both
listeners default to `0.0.0.0`, which accepts IPv4 connections only. The request plane and the direct
ZMQ event plane select an IPv6 address automatically when the host has no usable non-loopback IPv4
address.
</Note>

<ParamField path="--connector" type="string" default="null">
KV-cache transfer connector. Accepts zero or more values. Deprecated for vLLM — use `--kv-transfer-config` instead. For TRT-LLM, valid options are `nixl`, `lmcache`, `kvbm`, `null`, and `none`.

Expand Down Expand Up @@ -260,6 +341,15 @@ Unless a field is marked environment-only, it has both a CLI flag and an environ
effective per-worker cap is the engine limit plus this queue limit.
</ParamField>

<ParamField path="DYN_SYSTEM_HOST" type="string" default="0.0.0.0">
Listen address for the system status server on `DYN_SYSTEM_PORT`, which serves health, metrics,
and self-hosted model metadata. `0.0.0.0` accepts IPv4 connections only; on IPv6-only hosts, set
`::`. For a wildcard address, Dynamo advertises a non-loopback address from the same family; `::`
also accepts IPv4 connections, so it falls back to a non-loopback IPv4 address. If none exists,
Dynamo advertises loopback and logs a warning; remote callers such as the Frontend cannot reach
that address.
</ParamField>

## Related pages

<CardGroup cols={2}>
Expand Down
8 changes: 1 addition & 7 deletions lib/backend-common/src/rl.rs
Original file line number Diff line number Diff line change
Expand Up @@ -90,13 +90,7 @@ pub(crate) async fn serve_endpoint(

fn self_host_base_url(drt: &dynamo_runtime::DistributedRuntime) -> Option<String> {
let info = drt.system_status_server_info()?;
let socket_addr = info.socket_addr;
if socket_addr.ip().is_unspecified() {
let host = dynamo_runtime::utils::local_ip_for_advertise();
Some(format!("http://{host}:{}", socket_addr.port()))
} else {
Some(format!("http://{socket_addr}"))
}
Some(format!("http://{}", info.advertised_socket_addr()))
}

fn resolve_endpoint_name(primary_name: &str) -> anyhow::Result<String> {
Expand Down
2 changes: 2 additions & 0 deletions lib/bindings/kvbm/Cargo.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

2 changes: 2 additions & 0 deletions lib/bindings/python/Cargo.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

12 changes: 3 additions & 9 deletions lib/bindings/python/rust/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -1294,15 +1294,9 @@ impl DistributedRuntime {
/// Workers use this in their RL request-plane route descriptor so the
/// frontend does not need to derive worker system URLs from static env vars.
fn system_status_server_url(&self) -> Option<String> {
self.inner.system_status_server_info().map(|info| {
let socket_addr = info.socket_addr;
if socket_addr.ip().is_unspecified() {
let host = dynamo_runtime::utils::ip_resolver::local_ip_for_advertise();
format!("http://{host}:{}", socket_addr.port())
} else {
format!("http://{socket_addr}")
}
})
self.inner
.system_status_server_info()
.map(|info| format!("http://{}", info.advertised_socket_addr()))
}

/// Register an async Python callback for /engine/{route_name}
Expand Down
47 changes: 47 additions & 0 deletions lib/kv-router/src/services/common/zmq.rs
Original file line number Diff line number Diff line change
Expand Up @@ -59,6 +59,7 @@ impl ZmqSocket {
}

pub(crate) fn connect(&self, endpoint: &str) -> Result<()> {
set_ipv6_for(self.socket(), endpoint)?;
self.socket().connect(endpoint)?;
Ok(())
}
Expand Down Expand Up @@ -161,6 +162,13 @@ impl ZmqSocket {
}
}

/// libzmq resolves TCP hosts as IPv4 unless `ZMQ_IPV6` is set, and snapshots the option per
/// bind or connect. [`validate_endpoint`] rejects unbracketed IPv6 literals.
fn set_ipv6_for(socket: &zmq::Socket, endpoint: &str) -> Result<()> {
socket.set_ipv6(endpoint.starts_with("tcp://["))?;
Ok(())
}

fn configure_common_socket(socket: &zmq::Socket) -> Result<()> {
socket.set_linger(ZMQ_LINGER_MS)?;
socket.set_reconnect_ivl(ZMQ_RECONNECT_IVL_MS)?;
Expand Down Expand Up @@ -225,6 +233,7 @@ pub(crate) fn connect_sub_socket(endpoint: &str) -> Result<SharedSocket> {
pub(crate) fn connect_dealer_socket(endpoint: &str) -> Result<SharedSocket> {
Ok(Arc::new(Mutex::new(build_socket(zmq::DEALER, |socket| {
configure_bidirectional_socket(socket)?;
set_ipv6_for(socket, endpoint)?;
socket.connect(endpoint)?;
Ok(())
})?)))
Expand All @@ -238,6 +247,7 @@ pub(crate) fn connect_dealer_socket(endpoint: &str) -> Result<SharedSocket> {
pub(crate) fn create_bound_pub_socket(endpoint: &str) -> Result<ZmqSocket> {
build_socket(zmq::PUB, |socket| {
configure_send_socket(socket)?;
set_ipv6_for(socket, endpoint)?;
socket.bind(endpoint)?;
Ok(())
})
Expand Down Expand Up @@ -298,3 +308,40 @@ pub(crate) fn validate_endpoint(endpoint: &str) -> Result<()> {
)),
}
}

#[cfg(test)]
mod tests {
use super::*;
use std::time::Duration;

#[tokio::test]
async fn bracketed_ipv6_endpoints_connect() {
if let Err(error) = std::net::TcpListener::bind("[::1]:0") {
eprintln!("Skipping IPv6 ZMQ test: {error}");
return;
}
let mut publisher = create_bound_pub_socket("tcp://[::1]:*").unwrap();
let endpoint = publisher.socket().get_last_endpoint().unwrap().unwrap();
validate_endpoint(&endpoint).unwrap();
let mut subscriber = create_sub_socket(b"").unwrap();
subscriber.connect(&endpoint).unwrap();

tokio::time::timeout(Duration::from_secs(5), async {
loop {
publisher
.send_multipart(vec![b"ipv6".to_vec()])
.await
.unwrap();
if let Ok(frames) =
tokio::time::timeout(Duration::from_millis(50), subscriber.recv_multipart())
.await
{
assert_eq!(frames.unwrap(), vec![b"ipv6".to_vec()]);
break;
}
}
})
.await
.expect("IPv6 SUB socket should receive the message");
}
}
Loading
Loading