feat(sglang): relay node-local KV events from headless sidecars

lib/bindings/python/rust/backend.rs

@@ -108,10 +108,7 @@ fn sglang_sidecar_argv(argv: Vec<String>) -> Vec<String> {
 #[pyo3(signature = (argv=None))]
 fn _run_sglang_sidecar(py: Python<'_>, argv: Option<Vec<String>>) -> PyResult<()> {
     let cli_argv = sglang_sidecar_argv(argv.unwrap_or_default());
-    let bootstrap = dynamo_sglang_sidecar::SglangSidecarEngine::try_from_args_async(cli_argv)
-        .map_err(sidecar_startup_to_pyerr)?;
-
-    py.allow_threads(move || dynamo_sidecar_common::run(bootstrap))
+    py.allow_threads(move || dynamo_sglang_sidecar::run(cli_argv))
         .map_err(sidecar_run_to_pyerr)
 }
 

lib/sidecar/common/src/lib.rs

@@ -33,7 +33,7 @@ pub use transport::{
     DEFAULT_MAX_GRPC_MESSAGE_SIZE, GrpcChannelPool, format_error_chain, startup_deadline,
 };
 
-pub use run::run;
+pub use run::{run, run_task};
 
 #[cfg(test)]
 #[path = "transport/tests.rs"]

lib/sidecar/common/src/run.rs

@@ -16,6 +16,18 @@ use crate::SidecarStartupError;
 pub fn run<E: LLMEngine + 'static>(
     bootstrap: impl Future<Output = Result<(E, WorkerConfig), DynamoError>>,
 ) -> anyhow::Result<()> {
+    run_task(
+        |runtime, shutdown| async move { run_until_shutdown(bootstrap, &runtime, shutdown).await },
+    )
+}
+
+/// Run a sidecar task with the shared runtime and process-signal lifecycle.
+/// The task owns discovery and may serve inference or relay telemetry only.
+pub fn run_task<F, Fut>(task: F) -> anyhow::Result<()>
+where
+    F: FnOnce(Runtime, CancellationToken) -> Fut,
+    Fut: Future<Output = anyhow::Result<()>>,
+{
     logging::init();
     let runtime = Runtime::from_settings()?;
     let secondary = runtime.secondary();
@@ -35,7 +47,7 @@ pub fn run<E: LLMEngine + 'static>(
             signal_runtime.mark_shutting_down();
         });
 
-        let result = run_until_shutdown(bootstrap, &runtime, shutdown.clone()).await;
+        let result = task(runtime.clone(), shutdown.clone()).await;
         shutdown.cancel();
         signal_handle.abort();
         let _ = signal_handle.await;

lib/sidecar/sglang/Cargo.toml

@@ -21,6 +21,7 @@ path = "src/main.rs"
 
 [dependencies]
 dynamo-backend-common = { workspace = true }
+dynamo-llm = { path = "../../llm", default-features = false }
 dynamo-runtime = { workspace = true }
 dynamo-sidecar-common = { workspace = true }
 
@@ -31,6 +32,7 @@ clap = { version = "4", features = ["derive", "env"] }
 futures = { workspace = true }
 rand = { workspace = true }
 reqwest = { workspace = true }
+serde = { workspace = true }
 serde_json = { workspace = true }
 tokio = { workspace = true }
 tokio-util = { workspace = true }
@@ -45,4 +47,8 @@ tonic-build = { workspace = true }
 
 [dev-dependencies]
 dynamo-backend-common = { workspace = true, features = ["testing"] }
+dynamo-kv-router = { workspace = true }
+rmp-serde = { workspace = true }
+tempfile = { workspace = true }
+tmq = { workspace = true }
 tracing-subscriber = { workspace = true }

lib/sidecar/sglang/README.md

@@ -36,7 +36,7 @@ The sidecar discovers the model and tokenizer paths, served model name, parser d
 
 SGLang remains the source of truth for the worker's aggregated, prefill, or decode role. The inherited `--disaggregation-mode` option and `DYN_DISAGGREGATION_MODE` environment variable have no effect in this sidecar. The SGLang sidecar rejects `--route-to-encoder` because its native protocol does not support encoder workers. Disaggregated workers continue to register under their fixed role components; aggregated workers honor `--component` or `DYN_COMPONENT`.
 
-The sidecar opens eight gRPC connections by default. Override the pool size with `--grpc-connections` or `DYN_SIDECAR_GRPC_CONNECTIONS`.
+The full sidecar opens eight gRPC connections by default. Override the pool size with `--grpc-connections` or `DYN_SIDECAR_GRPC_CONNECTIONS`. Telemetry-only mode uses one metadata connection.
 
 Connection startup uses a 30-second timeout per attempt, a one-second retry and readiness interval, and a 30-minute deadline for establishing the full connection pool. Override them with `--grpc-connect-attempt-timeout-secs`, `--grpc-retry-interval-secs`, and `--grpc-startup-deadline-secs`, or with the corresponding `DYN_SIDECAR_GRPC_*` environment variables.
 
@@ -55,7 +55,47 @@ python3 -m sglang.launch_server \
 The entry point configures Dynamo logging when `main()` runs, then calls the
 private `dynamo._core.backend._run_sglang_sidecar(argv)` binding. The binding
 prepends the executable name expected by clap, releases the GIL, and runs the
-same unified worker lifecycle as the standalone executable.
+same mode dispatcher as the standalone executable.
+
+## Multinode DP and KV routing
+
+For multinode deployments with multiple data-parallel (DP) replicas and KV-aware
+routing, launch the sidecar on the leader (`node_rank=0`) and on each follower
+node that owns a DP scheduler/KV publisher. The sidecar automatically selects
+inference or telemetry-only mode from the local engine's `GetServerInfo` metadata.
+Each DP rank has its own KV cache; follower sidecars publish those local KV events
+to Dynamo without accepting inference requests.
+A TP-only follower that holds part of a replica but has no local KV publisher
+does not need a sidecar.
+
+Launch each sidecar separately with `--grpc-endpoint` pointing to its local
+SGLang server. This requires an SGLang build exposing node-local
+`kv_event_sources` through `GetServerInfo` on leaders and followers started
+with `--grpc-port`.
+
+For `launch/multinode_kv_router_sidecar.sh`, use an SGLang build including commit
+[`c50b251`](https://github.com/sgl-project/sglang/commit/c50b251da8e455dfbb594f8c3b7ccae6eca72129),
+which provides node-local KV source discovery and explicit publisher binding.
+The stock v0.5.19 image guidance below applies to the single-node examples.
+
+The multinode example also requires SGLang's explicit
+`"bind": true` KV-event option. It binds each rank's publisher to `127.0.0.1`, so
+the engine and its sidecar must share a network namespace. KV events can include
+token IDs and request metadata; loopback prevents remote access but does not
+isolate other processes in that namespace. If overriding this to a non-loopback
+bind, restrict every publisher port (`SGLANG_KV_EVENT_PORT + dp_rank`) to authorized
+consumers using a firewall or network policy.
+
+For disaggregated multinode attention DP, use
+`launch/multinode_disagg_kv_router_sidecar.sh prefill` on each prefill node and
+`launch/multinode_disagg_kv_router_sidecar.sh decode` on each decode node.
+Each role is a separate distributed engine group: give it its own
+`DIST_INIT_ADDR` and number its nodes from zero. The defaults require four
+nodes in total, with `NNODES=2`, `TP_SIZE=2`, and `DP_SIZE=2` per role.
+Set `SGLANG_BOOTSTRAP_HOST` to the reachable prefill leader address on prefill
+node zero. All nodes must share the Dynamo namespace, discovery and event
+services; start `python3 -m dynamo.frontend --router-mode kv` separately.
+Run the script with `--help` for the four per-node commands.
 
 ## Deploy on Kubernetes (quick start)
 

lib/sidecar/sglang/launch/multinode_disagg_kv_router_sidecar.sh

@@ -0,0 +1,36 @@
+#!/bin/bash
+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+#
+# One node of a multinode prefill or decode group with NIXL transfer and KV routing.
+# Defaults: two nodes per role, one GPU per node, Qwen/Qwen3-32B.
+
+set -e
+
+SCRIPT_DIR="$(dirname "$(readlink -f "$0")")"
+
+if [[ "${1:-}" == --help || "${1:-}" == -h ]]; then
+    echo "Usage: NODE_RANK=<rank> DIST_INIT_ADDR=<role-leader:port> bash $0 <prefill|decode> [SGLang options...]"
+    echo "Run once per node, with separate rendezvous addresses for prefill and decode."
+    echo "All nodes and the frontend must share Dynamo namespace, discovery and event services."
+    echo
+    echo "Example: run each command on its respective node (four nodes total):"
+    echo "  NODE_RANK=0 DIST_INIT_ADDR=prefill-leader:29500 SGLANG_BOOTSTRAP_HOST=prefill-leader bash $0 prefill"
+    echo "  NODE_RANK=1 DIST_INIT_ADDR=prefill-leader:29500 bash $0 prefill"
+    echo "  NODE_RANK=0 DIST_INIT_ADDR=decode-leader:29500 bash $0 decode"
+    echo "  NODE_RANK=1 DIST_INIT_ADDR=decode-leader:29500 bash $0 decode"
+    echo
+    echo "Start the frontend separately: python3 -m dynamo.frontend --router-mode kv"
+    echo "Use reachable leader addresses; NIXL requires connectivity between the two groups."
+    echo "For model, topology and port overrides: bash $SCRIPT_DIR/multinode_kv_router_sidecar.sh --help"
+    exit 0
+fi
+
+case "${1:-}" in
+    prefill|decode) export ROLE="$1"; shift ;;
+    *) echo "First argument must be prefill or decode; use --help for examples" >&2; exit 1 ;;
+esac
+
+# Reuse per-node engine/sidecar startup, validation, GPU budgeting and cleanup.
+# ROLE enables NIXL; node 0 serves requests and followers relay local KV events.
+exec bash "$SCRIPT_DIR/multinode_kv_router_sidecar.sh" "$@"

lib/sidecar/sglang/launch/multinode_kv_router_sidecar.sh

@@ -0,0 +1,114 @@
+#!/bin/bash
+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+#
+# One node of an externally managed SGLang attention-DP group with local KV events.
+# Requires node-local KV source discovery and this Dynamo sidecar on every node.
+# Start the frontend separately.
+# Defaults: two nodes, one GPU per node, global TP=2 / attention DP=2.
+
+set -e
+
+SCRIPT_DIR="$(dirname "$(readlink -f "$0")")"
+export DYNAMO_HOME="${DYNAMO_HOME:-$(readlink -f "$SCRIPT_DIR/../../../..")}"
+# shellcheck disable=SC1091 # Resolved relative to this script at runtime.
+source "$DYNAMO_HOME/examples/common/gpu_utils.sh"
+# shellcheck disable=SC1091 # Resolved relative to this script at runtime.
+source "$DYNAMO_HOME/examples/common/launch_utils.sh"
+
+if [[ "${1:-}" == --help || "${1:-}" == -h ]]; then
+    echo "Usage: NODE_RANK=<rank> DIST_INIT_ADDR=<leader-ip:port> bash $0 [SGLang options...]"
+    echo "Run once per node; use the same Dynamo namespace, discovery and event services."
+    echo "Required: NODE_RANK, DIST_INIT_ADDR"
+    echo "Topology: NNODES=2, TP_SIZE=NNODES, DP_SIZE=NNODES (global sizes; PP=1)"
+    echo "          DP_SIZE must be divisible by NNODES so every node owns publishers."
+    echo "Model:    MODEL=Qwen/Qwen3-32B, MAX_MODEL_LEN=4096, MAX_CONCURRENT_SEQS=32"
+    echo "Role:     ROLE=aggregated|prefill|decode (default: aggregated)"
+    echo "PD:       SGLANG_BOOTSTRAP_HOST=<reachable prefill leader IP> on prefill node 0"
+    echo "Ports:    SGLANG_HTTP_PORT=30000, SGLANG_GRPC_PORT=30001, SGLANG_KV_EVENT_PORT=5557"
+    echo "          SGLANG_DISAGGREGATION_BOOTSTRAP_PORT=8998, DYN_SYSTEM_PORT=8081"
+    echo "          DYN_HTTP_PORT=8000 is the separately launched frontend's port."
+    echo "Other:    SGLANG_PYTHON=python3, SGLANG_HOST=0.0.0.0, SGLANG_PAGE_SIZE=64"
+    echo "          CUDA_VISIBLE_DEVICES is inherited. Extra arguments go to SGLang only."
+    exit 0
+fi
+
+: "${NODE_RANK:?Set NODE_RANK to the local node rank within the engine group}"
+: "${DIST_INIT_ADDR:?Set DIST_INIT_ADDR to the reachable engine leader host:port}"
+NNODES="${NNODES:-2}"
+TP_SIZE="${TP_SIZE:-$NNODES}"
+DP_SIZE="${DP_SIZE:-$NNODES}"
+for size in "$NNODES" "$TP_SIZE" "$DP_SIZE"; do
+    if [[ ! "$size" =~ ^[1-9][0-9]*$ ]]; then
+        echo "NNODES, TP_SIZE and DP_SIZE must be positive integers" >&2
+        exit 1
+    fi
+done
+if [[ ! "$NODE_RANK" =~ ^(0|[1-9][0-9]*)$ ]] || (( NODE_RANK >= NNODES )); then
+    echo "NODE_RANK must be in [0, NNODES)" >&2
+    exit 1
+fi
+if (( NNODES < 2 || TP_SIZE % DP_SIZE != 0 || DP_SIZE % NNODES != 0 )); then
+    echo "This example requires NNODES >= 2, TP_SIZE divisible by DP_SIZE, and DP_SIZE divisible by NNODES" >&2
+    exit 1
+fi
+
+MODEL="${MODEL:-Qwen/Qwen3-32B}"
+ROLE="${ROLE:-aggregated}"
+SGLANG_PYTHON="${SGLANG_PYTHON:-python3}"
+SGLANG_HOST="${SGLANG_HOST:-0.0.0.0}"
+SGLANG_HTTP_PORT="${SGLANG_HTTP_PORT:-30000}"
+SGLANG_GRPC_PORT="${SGLANG_GRPC_PORT:-30001}"
+SGLANG_KV_EVENT_PORT="${SGLANG_KV_EVENT_PORT:-5557}"
+SGLANG_PAGE_SIZE="${SGLANG_PAGE_SIZE:-64}"
+MAX_MODEL_LEN="${MAX_MODEL_LEN:-4096}"
+MAX_CONCURRENT_SEQS="${MAX_CONCURRENT_SEQS:-32}"
+HTTP_PORT="${DYN_HTTP_PORT:-8000}"
+GPU_MEM_ARGS=$(build_sglang_gpu_mem_args)
+ENGINE_ROLE_ARGS=()
+SIDECAR_ARGS=()
+case "$SGLANG_HOST" in
+    0.0.0.0) SIDECAR_HOST=127.0.0.1 ;;
+    ::) SIDECAR_HOST='[::1]' ;;
+    *:*) SIDECAR_HOST="[$SGLANG_HOST]" ;;
+    *) SIDECAR_HOST="$SGLANG_HOST" ;;
+esac
+
+case "$ROLE" in
+    aggregated) ;;
+    prefill|decode)
+        ENGINE_ROLE_ARGS=(--disaggregation-mode "$ROLE" --disaggregation-transfer-backend nixl
+            --disaggregation-bootstrap-port "${SGLANG_DISAGGREGATION_BOOTSTRAP_PORT:-8998}")
+        if [[ "$ROLE" == prefill && "$NODE_RANK" == 0 ]]; then
+            : "${SGLANG_BOOTSTRAP_HOST:?Set SGLANG_BOOTSTRAP_HOST to the reachable prefill leader address}"
+            SIDECAR_ARGS+=(--bootstrap-host "$SGLANG_BOOTSTRAP_HOST")
+        fi
+        ;;
+    *) echo "ROLE must be aggregated, prefill or decode" >&2; exit 1 ;;
+esac
+print_launch_banner --no-curl "SGLang multinode KV sidecar ($ROLE, node $NODE_RANK)" "$MODEL" "$HTTP_PORT" \
+    "Topology: NNODES=$NNODES, TP_SIZE=$TP_SIZE, DP_SIZE=$DP_SIZE (attention DP)" \
+    "Rendezvous: $DIST_INIT_ADDR; local gRPC: $SGLANG_GRPC_PORT" \
+    "Start the frontend separately. Restart all nodes and sidecars together after failure."
+
+trap dynamo_exit_trap EXIT
+
+# Sidecars retry gRPC startup; no sleep or inference health check is needed on followers.
+# Bind KV events only on loopback; engine and sidecar share a network namespace.
+# shellcheck disable=SC2086 # GPU_MEM_ARGS intentionally expands into multiple flags.
+"$SGLANG_PYTHON" -m sglang.launch_server \
+    --model-path "$MODEL" \
+    --host "$SGLANG_HOST" --port "$SGLANG_HTTP_PORT" --grpc-port "$SGLANG_GRPC_PORT" \
+    --incremental-streaming-output \
+    --nnodes "$NNODES" --node-rank "$NODE_RANK" --dist-init-addr "$DIST_INIT_ADDR" \
+    --tp-size "$TP_SIZE" --dp-size "$DP_SIZE" --enable-dp-attention \
+    --kv-events-config "{\"publisher\":\"zmq\",\"endpoint\":\"tcp://127.0.0.1:${SGLANG_KV_EVENT_PORT}\",\"bind\":true,\"topic\":\"\"}" \
+    --page-size "$SGLANG_PAGE_SIZE" --context-length "$MAX_MODEL_LEN" \
+    --max-running-requests "$MAX_CONCURRENT_SEQS" \
+    "${ENGINE_ROLE_ARGS[@]}" $GPU_MEM_ARGS "$@" &
+
+DYN_SYSTEM_PORT="${DYN_SYSTEM_PORT:-8081}" \
+    python3 -m dynamo.sglang.sidecar \
+    --grpc-endpoint "http://$SIDECAR_HOST:$SGLANG_GRPC_PORT" "${SIDECAR_ARGS[@]}" &
+
+wait_any_exit