diff --git a/.github/filters.yaml b/.github/filters.yaml index 378b95d1c48b..e18cfef99e0a 100644 --- a/.github/filters.yaml +++ b/.github/filters.yaml @@ -195,6 +195,7 @@ sglang: - 'examples/backends/sglang/**' - 'components/src/dynamo/sglang/**' - 'components/src/dynamo/sglang_sidecar/**' + - 'lib/sglang-sidecar/**' - 'container/templates/sglang_*' - 'container/deps/sglang/**' - '!**/*.md' diff --git a/lib/sglang-sidecar/Cargo.toml b/lib/sglang-sidecar/Cargo.toml index 545430f707f4..11a495bad972 100644 --- a/lib/sglang-sidecar/Cargo.toml +++ b/lib/sglang-sidecar/Cargo.toml @@ -9,7 +9,7 @@ authors.workspace = true license.workspace = true homepage.workspace = true repository.workspace = true -description = "Dynamo SGLang sidecar backend — drives an out-of-process SGLang engine over its native gRPC service through the backend-common LLMEngine trait." +description = "Dynamo SGLang sidecar for an out-of-process engine exposed through its native gRPC service." [package.metadata.cargo-machete] # Referenced by the protobuf client generated into OUT_DIR at build time. diff --git a/lib/sglang-sidecar/README.md b/lib/sglang-sidecar/README.md new file mode 100644 index 000000000000..b06eec6403a2 --- /dev/null +++ b/lib/sglang-sidecar/README.md @@ -0,0 +1,53 @@ +# SGLang sidecar executable + +`dynamo-sglang-sidecar` connects Dynamo's unified worker lifecycle to an +out-of-process SGLang engine through SGLang's native gRPC service. + +Build and run it directly from the Dynamo workspace: + +```bash +cargo build --release -p dynamo-sglang-sidecar +./target/release/dynamo-sglang-sidecar \ + --sglang-endpoint http://127.0.0.1:30001 +``` + +Distribution and container packaging for the standalone executable are +intentionally deferred to a follow-up change. + +## SGLang-managed contract + +An SGLang launcher can supervise the executable directly after its native gRPC +listener is ready: + +```bash +python3 -m sglang.launch_server \ + \ + --grpc-port 30001 \ + --sidecar-executable dynamo-sglang-sidecar +``` + +The corresponding SGLang implementation should: + +1. Resolve the executable without invoking a shell. +2. Spawn `dynamo-sglang-sidecar --sglang-endpoint ` only after + native gRPC is listening. Preserve the parent environment so Dynamo's + namespace, discovery, and observability settings reach the worker. +3. Keep the executable as the directly supervised child. A spawn wrapper may + install SGLang's parent-death handling and then call `os.execvp`, which + preserves the child PID across the exec. +4. Treat an unexpected or non-zero child exit as fatal to the SGLang server. +5. On shutdown, send `SIGTERM`, wait for Dynamo's graceful lifecycle, and then + kill the remaining process tree if the deadline expires. + +The supervisor timeout must be configurable. Its default must exceed Dynamo's +combined release-mode shutdown budget: the 5-second +`DYN_GRACEFUL_SHUTDOWN_GRACE_PERIOD_SECS` default plus the 30-second +`DYN_WORKER_GRACEFUL_SHUTDOWN_TIMEOUT` default. A 40-second default leaves a +small supervision margin. Operators that increase either Dynamo value must +increase the SGLang timeout as well. + +SGLang's gRPC discovery response supplies the model identity and aggregated, +prefill, or decode role, so the managed executable needs only the endpoint +argument. Prefill deployments may additionally set +`SGLANG_DISAGGREGATION_BOOTSTRAP_HOST` when the discovered address is not +routable from decode workers. diff --git a/lib/sglang-sidecar/src/args.rs b/lib/sglang-sidecar/src/args.rs index d488788b2c38..ba03fcb548be 100644 --- a/lib/sglang-sidecar/src/args.rs +++ b/lib/sglang-sidecar/src/args.rs @@ -1,7 +1,7 @@ // SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -//! Command-line arguments and transport configuration for the SGLang gRPC sidecar. +//! Command-line arguments and transport configuration for the SGLang sidecar. use std::path::PathBuf; use std::time::Duration; @@ -10,7 +10,7 @@ use std::time::Duration; #[derive(clap::Parser, Debug, Clone)] #[command( name = "dynamo-sglang-sidecar", - about = "Dynamo SGLang sidecar — drives an out-of-process SGLang native gRPC server." + about = "Dynamo sidecar for an out-of-process SGLang native gRPC server." )] pub struct Args { /// `host:port` (or URL) of SGLang's native `sglang.runtime.v1` service. @@ -120,7 +120,19 @@ pub fn normalize_endpoint(raw: &str) -> Result { #[cfg(test)] mod tests { - use super::normalize_endpoint; + use super::{Args, normalize_endpoint}; + use clap::Parser; + + #[test] + fn parses_sglang_managed_executable_args() { + let args = Args::try_parse_from([ + "dynamo-sglang-sidecar", + "--sglang-endpoint", + "http://127.0.0.1:30001", + ]) + .unwrap(); + assert_eq!(args.sglang_endpoint, "http://127.0.0.1:30001"); + } #[test] fn normalizes_bare_and_grpc_endpoints() { diff --git a/lib/sglang-sidecar/src/engine.rs b/lib/sglang-sidecar/src/engine.rs index f46fb4d15265..9cfe2e020be7 100644 --- a/lib/sglang-sidecar/src/engine.rs +++ b/lib/sglang-sidecar/src/engine.rs @@ -63,6 +63,10 @@ impl SglangSidecarEngine { } .map_err(|err| client::invalid_arg(err.to_string()))?; + Self::from_parsed_args(args) + } + + pub fn from_parsed_args(args: Args) -> Result<(Self, WorkerConfig), DynamoError> { let endpoint = normalize_endpoint(&args.sglang_endpoint).map_err(client::invalid_arg)?; let transport = args.transport(); let discovery = bootstrap_discover(&endpoint, &transport)?; diff --git a/lib/sglang-sidecar/src/main.rs b/lib/sglang-sidecar/src/main.rs index fe592db37c83..e020158b6e6c 100644 --- a/lib/sglang-sidecar/src/main.rs +++ b/lib/sglang-sidecar/src/main.rs @@ -3,13 +3,18 @@ //! Entry point for the `dynamo-sglang-sidecar` binary. //! -//! Mirrors the mocker backend: bootstrap-discover the engine in `from_args` -//! (building the [`WorkerConfig`](dynamo_backend_common::WorkerConfig) `run` -//! needs synchronously), then hand the engine to the shared runtime harness. +//! Mirrors the mocker backend: parse the process CLI with clap, bootstrap-discover +//! the engine in `from_parsed_args` (building the +//! [`WorkerConfig`](dynamo_backend_common::WorkerConfig) `run` needs +//! synchronously), then hand the engine to the shared runtime harness. use std::sync::Arc; +use clap::Parser; +use dynamo_sglang_sidecar::args::Args; + fn main() -> anyhow::Result<()> { - let (engine, config) = dynamo_sglang_sidecar::SglangSidecarEngine::from_args(None)?; + let args = Args::parse(); + let (engine, config) = dynamo_sglang_sidecar::SglangSidecarEngine::from_parsed_args(args)?; dynamo_backend_common::run(Arc::new(engine), config) } diff --git a/lib/sglang-sidecar/tests/executable.rs b/lib/sglang-sidecar/tests/executable.rs new file mode 100644 index 000000000000..fa4e5003e92a --- /dev/null +++ b/lib/sglang-sidecar/tests/executable.rs @@ -0,0 +1,21 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +use std::process::Command; + +#[test] +fn executable_exposes_sglang_managed_contract() { + let output = Command::new(env!("CARGO_BIN_EXE_dynamo-sglang-sidecar")) + .arg("--help") + .output() + .expect("run dynamo-sglang-sidecar --help"); + + assert!( + output.status.success(), + "--help failed: {}", + String::from_utf8_lossy(&output.stderr) + ); + let stdout = String::from_utf8(output.stdout).expect("help output is UTF-8"); + assert!(stdout.contains("--sglang-endpoint")); + assert!(stdout.contains("SGLANG_GRPC_ENDPOINT")); +}