Skip to content
Merged
Show file tree
Hide file tree
Changes from 15 commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
22 changes: 22 additions & 0 deletions Cargo.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

1 change: 1 addition & 0 deletions Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@ members = [
"lib/backend-common",
"lib/backend-common/examples/mocker",
"lib/vllm-rs-backend",
"lib/sglang-sidecar",
"lib/bindings/c",
"lib/bindings/python/codegen",
"deploy/inference-gateway/ext-proc",
Expand Down
57 changes: 57 additions & 0 deletions examples/backends/sglang/launch/sidecar_agg.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,57 @@
#!/bin/bash
# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
# SPDX-License-Identifier: Apache-2.0
#
# Aggregated serving through SGLang's native gRPC server (1 GPU).
# Requires an SGLang build containing PR #23508 and, until it merges, PR #25185.

set -e
trap 'echo Cleaning up...; kill 0' EXIT

SCRIPT_DIR="$(dirname "$(readlink -f "$0")")"
source "$SCRIPT_DIR/../../../common/gpu_utils.sh"
source "$SCRIPT_DIR/../../../common/launch_utils.sh"

MODEL="Qwen/Qwen3-0.6B"
EXTRA_ARGS=()
while [[ $# -gt 0 ]]; do
case $1 in
--model-path)
MODEL="$2"
shift 2
;;
-h|--help)
echo "Usage: $0 [--model-path <name>] [SGLang options...]"
exit 0
;;
*)
EXTRA_ARGS+=("$1")
shift
;;
esac
done

SGLANG_HOST="${SGLANG_HOST:-127.0.0.1}"
SGLANG_HTTP_PORT="${SGLANG_HTTP_PORT:-30000}"
SGLANG_GRPC_PORT="${SGLANG_GRPC_PORT:-30001}"
HTTP_PORT="${DYN_HTTP_PORT:-8000}"
GPU_MEM_ARGS=$(build_sglang_gpu_mem_args)

print_launch_banner "Launching SGLang Native-gRPC Sidecar (1 GPU)" "$MODEL" "$HTTP_PORT"

python3 -m dynamo.frontend &

# --grpc-port enables the native Rust gRPC server alongside SGLang's HTTP API.
python3 -m sglang.launch_server \
Comment thread
connorcarpenter15 marked this conversation as resolved.
Resolved
--model-path "$MODEL" \
--host "$SGLANG_HOST" \
--port "$SGLANG_HTTP_PORT" \
--grpc-port "$SGLANG_GRPC_PORT" \
$GPU_MEM_ARGS \
"${EXTRA_ARGS[@]}" &

DYN_SYSTEM_PORT=${DYN_SYSTEM_PORT:-8081} \
dynamo-sglang-sidecar \
--sglang-endpoint "${SGLANG_HOST}:${SGLANG_GRPC_PORT}" &

wait_any_exit
95 changes: 95 additions & 0 deletions examples/backends/sglang/launch/sidecar_disagg.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,95 @@
#!/bin/bash
# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
# SPDX-License-Identifier: Apache-2.0
#
# Disaggregated serving through two SGLang native gRPC servers (2 GPUs).
# Requires SGLang PR #25185 plus server-side DisaggregatedParams forwarding.

set -e

SCRIPT_DIR="$(dirname "$(readlink -f "$0")")"
source "$SCRIPT_DIR/../../../common/gpu_utils.sh"
source "$SCRIPT_DIR/../../../common/launch_utils.sh"

MODEL="Qwen/Qwen3-0.6B"
EXTRA_ARGS=()
while [[ $# -gt 0 ]]; do
case $1 in
--model-path)
if [[ $# -lt 2 || "$2" == -* ]]; then
echo "Missing value for --model-path"
echo "Use --help for usage information"
exit 1
fi
MODEL="$2"
shift 2
;;
-h|--help)
echo "Usage: $0 [--model-path <name>] [SGLang options...]"
echo
echo "Set SGLANG_PYTHON to the Python executable containing the patched SGLang build."
exit 0
;;
*)
EXTRA_ARGS+=("$1")
shift
;;
esac
done

trap 'echo Cleaning up...; kill 0' EXIT

SGLANG_PYTHON="${SGLANG_PYTHON:-python3}"
SGLANG_HOST="${SGLANG_HOST:-127.0.0.1}"
SGLANG_BOOTSTRAP_HOST="${SGLANG_BOOTSTRAP_HOST:-$SGLANG_HOST}"
SGLANG_DISAGGREGATION_BOOTSTRAP_PORT="${SGLANG_DISAGGREGATION_BOOTSTRAP_PORT:-8998}"
SGLANG_PREFILL_HTTP_PORT="${SGLANG_PREFILL_HTTP_PORT:-30000}"
SGLANG_PREFILL_GRPC_PORT="${SGLANG_PREFILL_GRPC_PORT:-30001}"
SGLANG_DECODE_HTTP_PORT="${SGLANG_DECODE_HTTP_PORT:-30010}"
SGLANG_DECODE_GRPC_PORT="${SGLANG_DECODE_GRPC_PORT:-30011}"
SGLANG_PREFILL_GPU="${SGLANG_PREFILL_GPU:-0}"
SGLANG_DECODE_GPU="${SGLANG_DECODE_GPU:-1}"
HTTP_PORT="${DYN_HTTP_PORT:-8000}"

GPU_MEM_ARGS=$(build_sglang_gpu_mem_args)

print_launch_banner "Launching SGLang Native-gRPC Disaggregated Serving (2 GPUs)" "$MODEL" "$HTTP_PORT"

python3 -m dynamo.frontend &

CUDA_VISIBLE_DEVICES="$SGLANG_PREFILL_GPU" \
"$SGLANG_PYTHON" -m sglang.launch_server \
--model-path "$MODEL" \
--host "$SGLANG_HOST" \
--port "$SGLANG_PREFILL_HTTP_PORT" \
--grpc-port "$SGLANG_PREFILL_GRPC_PORT" \
--disaggregation-mode prefill \
--disaggregation-bootstrap-port "$SGLANG_DISAGGREGATION_BOOTSTRAP_PORT" \
--disaggregation-transfer-backend nixl \
$GPU_MEM_ARGS \
"${EXTRA_ARGS[@]}" &

CUDA_VISIBLE_DEVICES="$SGLANG_DECODE_GPU" \
"$SGLANG_PYTHON" -m sglang.launch_server \
--model-path "$MODEL" \
--host "$SGLANG_HOST" \
--port "$SGLANG_DECODE_HTTP_PORT" \
--grpc-port "$SGLANG_DECODE_GRPC_PORT" \
--disaggregation-mode decode \
--disaggregation-bootstrap-port "$SGLANG_DISAGGREGATION_BOOTSTRAP_PORT" \
--disaggregation-transfer-backend nixl \
$GPU_MEM_ARGS \
"${EXTRA_ARGS[@]}" &

OTEL_SERVICE_NAME=dynamo-worker-prefill \
DYN_SYSTEM_PORT="${DYN_SYSTEM_PORT1:-8081}" \
dynamo-sglang-sidecar \
--sglang-endpoint "${SGLANG_HOST}:${SGLANG_PREFILL_GRPC_PORT}" \
--bootstrap-host "$SGLANG_BOOTSTRAP_HOST" &

OTEL_SERVICE_NAME=dynamo-worker-decode \
DYN_SYSTEM_PORT="${DYN_SYSTEM_PORT2:-8082}" \
dynamo-sglang-sidecar \
--sglang-endpoint "${SGLANG_HOST}:${SGLANG_DECODE_GRPC_PORT}" &

wait_any_exit
3 changes: 2 additions & 1 deletion lib/backend-common/src/engine.rs
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,8 @@ pub use dynamo_llm::protocols::common::llm_backend::{
LLMEngineOutput, LogProbs, TopLogprob, TopLogprobs,
};
pub use dynamo_llm::protocols::common::preprocessor::{
BootstrapInfo, PrefillResult, PreprocessedRequest,
BootstrapInfo, MultimodalData, MultimodalDataMap, PrefillResult, PreprocessedRequest,
RoutingHints,
};
pub use dynamo_llm::protocols::common::{
FinishReason, GuidedDecodingOptions, OutputOptions, SamplingOptions, StopConditions,
Expand Down
49 changes: 49 additions & 0 deletions lib/sglang-sidecar/Cargo.toml
Original file line number Diff line number Diff line change
@@ -0,0 +1,49 @@
# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
# SPDX-License-Identifier: Apache-2.0

[package]
name = "dynamo-sglang-sidecar"
version.workspace = true
edition.workspace = true
authors.workspace = true
license.workspace = true
homepage.workspace = true
repository.workspace = true
description = "Dynamo SGLang sidecar backend — drives an out-of-process SGLang engine over its native gRPC service through the backend-common LLMEngine trait."

[package.metadata.cargo-machete]
# Referenced by the protobuf client generated into OUT_DIR at build time.
ignored = ["prost"]

[[bin]]
name = "dynamo-sglang-sidecar"
path = "src/main.rs"

[dependencies]
dynamo-backend-common = { workspace = true }
dynamo-runtime = { workspace = true }

anyhow = { workspace = true }
async-stream = { workspace = true }
async-trait = { workspace = true }
clap = { version = "4", features = ["derive", "env"] }
futures = { workspace = true }
rand = { workspace = true }
serde_json = { workspace = true }
tokio = { workspace = true }
tokio-util = { workspace = true }
tracing = { workspace = true }
url = { workspace = true }

# tonic / prost are pinned per-crate (not workspace deps): tonic 0.13.1
# depends on prost 0.13.5, matching `lib/llm`. tonic-build 0.13.1 still
# bundles prost codegen via `compile_protos` (the 0.14 split to
# tonic-prost-build does not apply here).
prost = { version = "0.13.5" }
tonic = { version = "0.13.1" }

[build-dependencies]
tonic-build = { version = "0.13.1" }

[dev-dependencies]
dynamo-backend-common = { workspace = true, features = ["testing"] }
24 changes: 24 additions & 0 deletions lib/sglang-sidecar/build.rs
Original file line number Diff line number Diff line change
@@ -0,0 +1,24 @@
// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
// SPDX-License-Identifier: Apache-2.0

//! Compiles client stubs from the temporarily vendored SGLang gRPC contract.

use std::env;
use std::path::PathBuf;

fn main() -> Result<(), Box<dyn std::error::Error>> {
let manifest_dir = PathBuf::from(env::var("CARGO_MANIFEST_DIR")?);
let proto_dir = manifest_dir.join("proto");
let proto_path = proto_dir.join("sglang.proto");

// `--experimental_allow_proto3_optional` lets older protoc (the container's
// apt protoc 3.12) compile the proto3 `optional` fields in sglang.proto
// (e.g. `routed_dp_rank`). The flag was added in protoc 3.12 for exactly
// this and is a no-op on 3.15+ where proto3 optional is stable.
tonic_build::configure()
.protoc_arg("--experimental_allow_proto3_optional")
.compile_protos(&[proto_path.as_path()], &[proto_dir.as_path()])?;

println!("cargo:rerun-if-changed={}", proto_path.display());
Ok(())
}
19 changes: 19 additions & 0 deletions lib/sglang-sidecar/proto/README.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,19 @@
<!--
SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
SPDX-License-Identifier: Apache-2.0
-->

# Temporary SGLang gRPC contract

This copy is temporary while Dynamo waits for SGLang to include
`sglang/srt/grpc/sglang.proto` in a release wheel. Once the contract is
available there, Dynamo should remove this directory, pin and install the
matching `sglang` wheel as a build dependency, and compile the packaged proto
instead.

The contract was copied from SGLang commit
[`cc7d6659fd68694797892d0d863b2549a5b61b69`](https://github.com/sgl-project/sglang/blob/cc7d6659fd68694797892d0d863b2549a5b61b69/proto/sglang/runtime/v1/sglang.proto).
The upstream file's SHA-256 is
`a2e14952ddb2b34b6e22cbbc4e76d76d70c44f2dbf087cb9918aed3399d9ef42`.
The local file adds SPDX and temporary-copy comments and applies Dynamo's
`clang-format` style; these changes do not alter the protobuf descriptor.
Loading
Loading