diff --git a/components/src/dynamo/sglang/CLAUDE.md b/components/src/dynamo/sglang/CLAUDE.md index 9fa2b4642bb7..27cb17d70839 100644 --- a/components/src/dynamo/sglang/CLAUDE.md +++ b/components/src/dynamo/sglang/CLAUDE.md @@ -313,10 +313,11 @@ text-to-video-diffusion.sh # 1-2 GPUs - Text-to-video (Wan2.1) Always slice with an offset, don't assume per-chunk logprobs. - **Zombie GPU processes**: `sgl_diffusion::scheduler` spawns a child process that survives parent kill. Always check `nvidia-smi` after teardown. -- **Session radix cache**: SGLang 0.5.14+ provides session-aware radix ownership - without an opt-in flag. The handler passes `agent_context.session_id` to SGLang - as `session_params.id`. Agent KV hints are forwarded as metadata but are not - acted on by the SGLang backend. This path does not create router affinity. +- **Session identity**: SGLang 0.5.14 does not support passive session-aware radix + ownership. Do not pass `agent_context.session_id` as `session_params.id`; + SGLang treats that field as an explicit session lifecycle and rejects IDs that + were not created through `open_session`. Session headers remain available for + tracing and router affinity. For troubleshooting (CuDNN, config.json errors, OOM, disagg connectivity), see `docs/backends/sglang/sglang-examples.md#troubleshooting`. diff --git a/components/src/dynamo/sglang/request_handlers/handler_base.py b/components/src/dynamo/sglang/request_handlers/handler_base.py index 51d0af251f9d..825d7bfc46e0 100644 --- a/components/src/dynamo/sglang/request_handlers/handler_base.py +++ b/components/src/dynamo/sglang/request_handlers/handler_base.py @@ -669,9 +669,6 @@ def __init__( self.serving_mode = config.serving_mode self.use_sglang_tokenizer = config.dynamo_args.use_sglang_tokenizer self.enable_trace = getattr(config.server_args, "enable_trace", False) - self.enable_session_radix_cache = getattr( - config.server_args, "enable_session_radix_cache", False - ) if engine is not None: self.input_param_manager = InputParamManager( @@ -1019,16 +1016,6 @@ def _get_input_param(self, request: Dict[str, Any]) -> Dict[str, Any]: "prompt" if isinstance(request_input, str) else "input_ids": request_input } - def _session_id(self, request: Dict[str, Any]) -> Optional[str]: - if not self.enable_session_radix_cache: - return None - session_id = (request.get("agent_context") or {}).get("session_id") - return session_id if isinstance(session_id, str) and session_id else None - - def _session_kwargs(self, request: Dict[str, Any]) -> Dict[str, Any]: - session_id = self._session_id(request) - return {"session_params": {"id": session_id}} if session_id else {} - @staticmethod def _get_guided_decoding_params( guided_decoding: Optional[Dict[str, Any]], diff --git a/components/src/dynamo/sglang/request_handlers/llm/decode_handler.py b/components/src/dynamo/sglang/request_handlers/llm/decode_handler.py index 23c04210acc0..a2eeb1220235 100644 --- a/components/src/dynamo/sglang/request_handlers/llm/decode_handler.py +++ b/components/src/dynamo/sglang/request_handlers/llm/decode_handler.py @@ -395,7 +395,6 @@ async def generate( external_trace_header=trace_header, rid=trace_id, data_parallel_rank=dp_rank, - **self._session_kwargs(request), lora_path=lora_path, **logprob_kwargs, **self._priority_kwargs(priority), @@ -463,7 +462,6 @@ async def generate( external_trace_header=trace_header, rid=trace_id, data_parallel_rank=dp_rank, - **self._session_kwargs(request), lora_path=lora_path, **logprob_kwargs, **self._priority_kwargs(priority), diff --git a/components/src/dynamo/sglang/request_handlers/llm/prefill_handler.py b/components/src/dynamo/sglang/request_handlers/llm/prefill_handler.py index cd6faae905cb..5c2045572e0f 100644 --- a/components/src/dynamo/sglang/request_handlers/llm/prefill_handler.py +++ b/components/src/dynamo/sglang/request_handlers/llm/prefill_handler.py @@ -166,7 +166,6 @@ async def generate( external_trace_header=trace_header, rid=trace_id, data_parallel_rank=dp_rank, - **self._session_kwargs(inner_request), lora_path=lora_path, **self._priority_kwargs(priority), ) diff --git a/components/src/dynamo/sglang/tests/test_sglang_frontend_decoding.py b/components/src/dynamo/sglang/tests/test_sglang_frontend_decoding.py index f174151ef933..ef412faa656d 100644 --- a/components/src/dynamo/sglang/tests/test_sglang_frontend_decoding.py +++ b/components/src/dynamo/sglang/tests/test_sglang_frontend_decoding.py @@ -97,7 +97,6 @@ async def no_cancellation_monitor(*args, **kwargs): handler._get_input_param = lambda req: {"input_ids": req.get("token_ids", [])} handler._resolve_lora = lambda req: None - handler._session_kwargs = lambda req: {} handler._priority_kwargs = lambda priority: {} return handler diff --git a/docs/backends/sglang/agents.md b/docs/backends/sglang/agents.md index 02d222f1f1e8..5735e1d97fc5 100644 --- a/docs/backends/sglang/agents.md +++ b/docs/backends/sglang/agents.md @@ -2,10 +2,10 @@ # SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 title: SGLang for Agentic Workloads -subtitle: Priority scheduling and session-aware radix KV for agentic serving +subtitle: Priority scheduling and KV cache tuning for agentic serving --- -This guide covers SGLang-specific configuration for agentic serving with Dynamo. It explains which SGLang engine flags to enable, how Dynamo's [agent hints](../../components/frontend/nvext.md#agent-hints) map to SGLang behavior, and how to tag radix KV by session. +This guide covers SGLang-specific configuration for agentic serving with Dynamo. It explains which SGLang engine flags to enable, how Dynamo's [agent hints](../../components/frontend/nvext.md#agent-hints) map to SGLang behavior, and how session headers interact with SGLang 0.5.14. ## Overview @@ -111,23 +111,14 @@ for chunk in response: print(chunk.choices[0].delta.content, end="") ``` -## Session Radix Cache +## Session Identity -SGLang can tag ordinary evictable radix KV with the normalized agent session ID without pinning requests to a worker or creating a separate streaming-session lifecycle. +Dynamo normalizes agent headers such as `X-Dynamo-Session-ID` for request tracing and router affinity. SGLang 0.5.14 does not support passive session-aware radix ownership, so the Dynamo worker does not attach this ID to SGLang generate requests. -> **Availability:** Session-aware radix ownership is built into SGLang 0.5.14 and later; no opt-in server argument is required. +> [!NOTE] +> SGLang 0.5.14's `session_params` belongs to its explicit session lifecycle and requires a session created through `open_session`. It is not a passive KV ownership tag. -Launch the worker with: - -```bash -python -m dynamo.sglang \ - --model-path \ - --radix-eviction-policy priority -``` - -Dynamo reads session identity from agent headers such as `X-Dynamo-Session-ID` and passes it to SGLang on every generate request. `X-Dynamo-Session-Final: true` is normalized into an internal KV eviction hint and forwarded with the agent context, but the SGLang backend does not act on that hint in this release. - -The radix entries remain normally evictable. Session-aware radix ownership does not create router affinity; a configured router can independently use `X-Dynamo-Session-ID` for router-local affinity. +The `--radix-eviction-policy priority` flag controls priority-based KV eviction only; it does not tag radix entries by session. `X-Dynamo-Session-Final: true` is normalized into an internal KV eviction hint, but the SGLang backend does not act on that hint in this release. ## Quickstart @@ -138,7 +129,7 @@ bash examples/backends/sglang/launch/agg_agent.sh \ --model-path zai-org/GLM-4.7-Flash --tp 2 ``` -Agent providers send session headers directly; no body-level lifecycle object is needed. +Agent providers send session headers directly for tracing and router affinity; no body-level lifecycle object is needed. ## See Also diff --git a/docs/backends/sglang/sglang-chat-processor.md b/docs/backends/sglang/sglang-chat-processor.md index 02330b227954..ba75ac87ac97 100644 --- a/docs/backends/sglang/sglang-chat-processor.md +++ b/docs/backends/sglang/sglang-chat-processor.md @@ -151,4 +151,4 @@ Key differences: - **[Tool Calling](../../tool-calling/README.md)**: General tool calling guide - **[Reference Guide](sglang-reference-guide.md)**: Full SGLang backend reference -- **[Agentic Workloads](agents.md)**: Priority scheduling and session-aware radix KV +- **[Agentic Workloads](agents.md)**: Priority scheduling and KV cache tuning diff --git a/docs/components/frontend/nvext.md b/docs/components/frontend/nvext.md index ad7b41457537..7b9cfabafc64 100644 --- a/docs/components/frontend/nvext.md +++ b/docs/components/frontend/nvext.md @@ -233,4 +233,4 @@ When the client requests response metadata via `extra_fields`, the response incl | [Session IDs](../../agents/session-ids.md) | Passive session identity | | [Agent Tracing](../../agents/agent-tracing.md) | JSONL request traces, inferred tool-call metadata, and harness tool-event ingestion | | [Agent Hints](../../agents/agent-hints.md) | Per-request serving hints for routing, scheduling, and cache behavior | -| [SGLang for Agentic Workloads](../../backends/sglang/agents.md) | SGLang engine flags for priority scheduling, eviction policies, and session-aware radix tagging | +| [SGLang for Agentic Workloads](../../backends/sglang/agents.md) | SGLang engine flags for priority scheduling and KV eviction policies | diff --git a/docs/digest/agentic-inference/agentic-inference.md b/docs/digest/agentic-inference/agentic-inference.md index f7e4df62d988..ff319bd4c1ac 100644 --- a/docs/digest/agentic-inference/agentic-inference.md +++ b/docs/digest/agentic-inference/agentic-inference.md @@ -170,7 +170,7 @@ Making blocks globally available solves the sharing problem, but does not solve The general design pattern is to attach zero or more retention directives to a request or token range. Blocks without directives follow the default LRU path with zero overhead. The evictor becomes a two-structure system: an LRU free list for unprioritized blocks (O(1), unchanged) and a priority queue for annotated blocks. Dynamo's public agent surface exposes the priority part today through `nvext.agent_hints.priority`; TTL or per-token-range retention directives are future API work. -Anthropic's prompt caching lets you mark prefixes as cacheable on their infrastructure. Dynamo does not currently expose the same semantics as a self-hosted `nvext.cache_control` TTL pinning API. The supported production path is priority-driven: `nvext.agent_hints.priority` can influence router queueing and, when the backend enables it, engine scheduling and priority-aware cache eviction. SGLang can additionally tag ordinary evictable radix KV by session. +Anthropic's prompt caching lets you mark prefixes as cacheable on their infrastructure. Dynamo does not currently expose the same semantics as a self-hosted `nvext.cache_control` TTL pinning API. The supported production path is priority-driven: `nvext.agent_hints.priority` can influence router queueing and, when the backend enables it, engine scheduling and priority-aware cache eviction. The next step is connecting richer retention directives with the distributed cache. Today, priority and session metadata are local serving signals, not a cluster-wide per-block TTL lease. Extending retention semantics across HiCache/KVBM's shared storage tier would let the harness mark a block once and have its priority, lifetime, and placement intent travel with it through the write-through path. Combined with the prefetch hooks described above, this gives the harness end-to-end lifecycle control across the full memory hierarchy. diff --git a/examples/backends/sglang/launch/_test_agg.sh b/examples/backends/sglang/launch/_test_agg.sh index 4113e9d2b618..449023e201bd 100755 --- a/examples/backends/sglang/launch/_test_agg.sh +++ b/examples/backends/sglang/launch/_test_agg.sh @@ -8,7 +8,6 @@ # # Same shape as agg_router.sh (2 workers, KV routing, KV events) but with: # - --enable-hierarchical-cache -> HiCache row populates -# - session-aware radix ownership -> session-tagged radix KV # - --enable-metrics-for-all-schedulers -> per-scheduler metrics # - --enable-mfu-metrics -> model FLOPs utilization # - --mem-fraction-static 0.92 -> larger KV pool per worker @@ -104,7 +103,6 @@ Topology: Features enabled for full Grafana coverage: - hierarchical KV cache (host RAM tier) -> HiCache row - - session radix cache -> session-tagged evictable KV - per-scheduler metrics + MFU metrics - mem-fraction-static=$MEM_FRACTION, max-running-requests=$MAX_RUNNING - page-size=$PAGE_SIZE, chunked-prefill-size=$CHUNKED_PREFILL diff --git a/examples/backends/sglang/launch/agg_agent.sh b/examples/backends/sglang/launch/agg_agent.sh index a7b502494a45..281cdc1ff7e2 100755 --- a/examples/backends/sglang/launch/agg_agent.sh +++ b/examples/backends/sglang/launch/agg_agent.sh @@ -2,7 +2,7 @@ # SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 # -# Aggregated agent serving with session radix tagging, KV event tracking, +# Aggregated agent serving with priority-based radix eviction, KV event tracking, # and reasoning/tool-call parsing. # GPUs: 2 (default model uses --tp 2) @@ -64,7 +64,7 @@ python3 -m dynamo.frontend \ --router-reset-states \ --enable-anthropic-api & -# Session-aware radix ownership is built into SGLang 0.5.14+. +# Use priority-based radix eviction for agent requests. DYN_SYSTEM_PORT=${DYN_SYSTEM_PORT:-8081} \ python3 -m dynamo.sglang \ --model-path "$MODEL" \