Skip to content
Merged
Show file tree
Hide file tree
Changes from 17 commits
Commits
Show all changes
30 commits
Select commit Hold shift + click to select a range
398c543
feat(thunderagent): include hicache retention budget
ishandhanani Jul 1, 2026
d4e6fcb
docs(agents): update ThunderAgent HiCache and Pi repro
ishandhanani Jul 2, 2026
a9e199f
docs(agents): trim Pi harness quickstart
ishandhanani Jul 2, 2026
fa15c33
docs(agents): render ThunderAgent Pi repro
ishandhanani Jul 2, 2026
cc40872
docs(agents): add ThunderAgent Pi launch steps
ishandhanani Jul 2, 2026
db9c368
docs(agents): add Harbor Pi benchmark recipe
ishandhanani Jul 2, 2026
96cd892
docs(agents): clarify Harbor run environment
ishandhanani Jul 2, 2026
770385c
docs(thunderagent): add Harbor Pi A/B walkthrough
ishandhanani Jul 3, 2026
9832697
docs(thunderagent): simplify Harbor Pi walkthrough
ishandhanani Jul 4, 2026
4bccbac
docs(thunderagent): focus Harbor benchmark steps
ishandhanani Jul 4, 2026
b31bfb2
fix(thunderagent): pin reproducible MiniMax setup
ishandhanani Jul 4, 2026
81c4948
docs(thunderagent): use cached Hugging Face model
ishandhanani Jul 4, 2026
afd3f49
docs(thunderagent): make Harbor scale setup reliable
ishandhanani Jul 4, 2026
b7e4216
fix(thunderagent): satisfy repro checks
ishandhanani Jul 4, 2026
97d7ac5
docs(thunderagent): require Docker Hub login
ishandhanani Jul 4, 2026
5a17e22
docs(thunderagent): fix online repro link
ishandhanani Jul 4, 2026
37bf8b1
docs(thunderagent): add reliable image pre-pull
ishandhanani Jul 4, 2026
4cabe9f
docs(thunderagent): address Harbor walkthrough review
ishandhanani Jul 4, 2026
3648136
docs(thunderagent): make session final explicit
ishandhanani Jul 4, 2026
ed42f0e
docs(agents): simplify Harbor and Pi setup
ishandhanani Jul 5, 2026
60de101
docs(thunderagent): clone Dynamo main in walkthrough
ishandhanani Jul 5, 2026
a2c7da9
ci(docs): ignore runner-hostile external links
ishandhanani Jul 5, 2026
da7984b
ci(docs): ignore additional unstable links
ishandhanani Jul 5, 2026
38553e4
ci(docs): keep PR link checks offline
ishandhanani Jul 5, 2026
ba61ce8
revert: remove Lychee CI workaround
ishandhanani Jul 5, 2026
925a50e
docs(agents): use Pi session identity by default
ishandhanani Jul 5, 2026
6ba54a6
docs(thunderagent): clarify router model name
ishandhanani Jul 5, 2026
13a636d
fix(thunderagent): stop launcher on child exit
ishandhanani Jul 5, 2026
5e5dfe2
fix(sglang): isolate HiCache capacity publication
ishandhanani Jul 5, 2026
394be5b
docs: follow Harbor adapter layout
ishandhanani Jul 5, 2026
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
18 changes: 18 additions & 0 deletions components/src/dynamo/sglang/register.py
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,7 @@
)

SGLANG_HICACHE_MOONCAKE_RUNTIME_KEY = "sglang_hicache_mooncake"
SGLANG_HICACHE_CAPACITY_RUNTIME_KEY = "sglang_hicache_capacity"
SPEC_DECODE_RUNTIME_KEY = "spec_decode"


Expand Down Expand Up @@ -340,6 +341,15 @@ def _eagle_enabled_for(speculative_algorithm: Optional[str]) -> bool:
return False


def _get_hicache_capacity_runtime_data(
scheduler_info: dict[str, Any],
) -> Optional[dict[str, int]]:
host_total_tokens = scheduler_info.get("hicache_host_total_tokens")
if not isinstance(host_total_tokens, (int, float)) or host_total_tokens <= 0:
return None
return {"host_total_tokens": int(host_total_tokens)}


async def _get_runtime_config(
engine: sgl.Engine, server_args: ServerArgs, dynamo_args: DynamoConfig
) -> Optional[ModelRuntimeConfig]:
Expand Down Expand Up @@ -454,6 +464,14 @@ async def _get_runtime_config(

try:
scheduler_info = engine._scheduler_init_result.scheduler_infos[0]
hicache_capacity_runtime_data = _get_hicache_capacity_runtime_data(
scheduler_info
)
if hicache_capacity_runtime_data is not None:
runtime_config.set_engine_specific(
SGLANG_HICACHE_CAPACITY_RUNTIME_KEY,
json.dumps(hicache_capacity_runtime_data),
)
Comment thread
ishandhanani marked this conversation as resolved.
Resolved
capacity = runtime_capacity(server_args, scheduler_info)
Comment thread
ishandhanani marked this conversation as resolved.
Resolved
max_total_tokens = scheduler_info.get("max_total_num_tokens")

Expand Down
17 changes: 17 additions & 0 deletions components/src/dynamo/sglang/tests/test_runtime_metadata.py
Original file line number Diff line number Diff line change
Expand Up @@ -70,3 +70,20 @@ def test_eagle_enabled_for_speculative_algorithm(speculative_algorithm, expected
from dynamo.sglang.register import _eagle_enabled_for

assert _eagle_enabled_for(speculative_algorithm) is expected


def test_hicache_capacity_runtime_data_uses_scheduler_capacity():
from dynamo.sglang.register import _get_hicache_capacity_runtime_data

assert _get_hicache_capacity_runtime_data({"hicache_host_total_tokens": 300}) == {
"host_total_tokens": 300
}


@pytest.mark.parametrize("value", [None, 0, -1, "300"])
def test_hicache_capacity_runtime_data_ignores_invalid_values(value):
from dynamo.sglang.register import _get_hicache_capacity_runtime_data

assert (
_get_hicache_capacity_runtime_data({"hicache_host_total_tokens": value}) is None
)
164 changes: 110 additions & 54 deletions components/src/dynamo/thunderagent_router/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -11,11 +11,7 @@ requests. It wraps Dynamo's native KV router and adds a program-level scheduler
with tool-boundary pause/resume, porting the scheduler from the ThunderAgent
paper.

**Conceptual docs live in [docs/agents/thunderagent-router.md](/docs/agents/thunderagent-router.md)** —
the scheduler model, the 5s scheduler tick (resume → pause), tool-boundary
pause/resume semantics, the utilization-driven control loop and its full knob
table, the architecture diagram, and the scheduler observability logs. This
README keeps only the build/run/repro specifics that belong next to the code.
**Conceptual docs live in [docs/agents/thunderagent-router.md](/docs/agents/thunderagent-router.md)** — the scheduler model, tool-boundary pause/resume semantics, the utilization-driven control loop, and observability. This README contains the source build and the complete Harbor/Pi A/B walkthrough.

## Install

Expand Down Expand Up @@ -73,6 +69,12 @@ Requests without session identity are passed through as one-off (no program
admission, no pause/resume). This is the safe fallback for non-agentic traffic
sharing the same workers.

### SGLang HiCache retention budget

`dynamo.sglang` publishes the authoritative GPU KV and HiCache host capacities in each worker's model deployment card. The scheduler automatically uses their sum as its retention budget, so `--pause-threshold 0.95` means 95% of the combined GPU + host pool; there is no ThunderAgent HiCache flag to set. This lets SGLang spill from GPU to its native host tier before ThunderAgent starts holding programs at tool boundaries.

Mooncake capacity is deliberately excluded. It is a content-addressed storage tier whose contents may be evicted or may not match the next request, not an unconditional program-retention budget. ThunderAgent does not call HiCache eviction, restore, prefetch, or Mooncake APIs; SGLang remains the admission and materialization authority.

## Tracing

Enable request tracing on the frontend with the master switch
Expand All @@ -94,70 +96,124 @@ at that endpoint (producers connect) and `tool_start` / `tool_end` /
`tool_call_id` pairs, giving you the full LLM-turn ↔ tool-gap timeline per
agent.

## Reproducing the MiniMax-M2 results
## Harbor/Pi A/B walkthrough

The headline numbers — program-aware scheduling vs KV-routing-only on the
same hardware — come from driving SWE-bench-Lite through **mini-SWE-agent** at
128 concurrent workers, against two TP4 MiniMax-M2 replicas on a single 8×H100
node.
This walkthrough runs the same SWE-bench Verified task through ThunderAgent and the stock Dynamo KV router. Harbor owns the task container, Pi runs inside it, and the model stack runs on the host. The example uses one 8-GPU node and two TP4 vLLM workers loading `MiniMaxAI/MiniMax-M2.7` from Hugging Face and serving the API alias `MiniMaxAI/MiniMax-M2`. There is no HiCache, Mooncake, shared cache, or frontend admission control in either arm.

### 1. Bring up Dynamo (2× TP4 MiniMax-M2)
The [agent-plugins `DynamoPi` adapter](https://github.com/ai-dynamo/agent-plugins/blob/main/pi-plugin/harbor_dynamo_pi.py) is required. It installs the Dynamo provider in each Harbor task container and maps Harbor's per-trial ID to one stable `x-dynamo-session-id` across all Pi turns. The ThunderAgent arm also sends one terminal session request so the router can release the completed program; the stock KV arm disables that request because it has no lifecycle consumer.

One script brings up both TP4 workers, the program-aware router, and the
frontend on `:8100`:
### 1. Install the three source trees

Use Python 3.12 and a machine with Docker and eight visible GPUs. Build Dynamo from the branch under test rather than installing a released wheel. Pi and its Node.js runtime are installed inside each Harbor task container.

```bash
bash components/src/dynamo/thunderagent_router/run_minimax_8xh100.sh
git clone https://github.com/ai-dynamo/dynamo ~/src/dynamo
git -C ~/src/dynamo fetch origin pull/11185/head
git -C ~/src/dynamo checkout --detach FETCH_HEAD

git clone --branch v0.16.0 --depth 1 https://github.com/harbor-framework/harbor ~/src/harbor
git clone https://github.com/ai-dynamo/agent-plugins ~/src/agent-plugins
git -C ~/src/agent-plugins checkout d0bae6a3c486c937f1b190ce21259a828cbaa8f9

cd ~/src/dynamo
uv venv --python 3.12 --seed .venv
source .venv/bin/activate
uv pip install pip 'maturin[patchelf]'
(cd lib/bindings/python && maturin develop --uv)
uv pip install -e lib/gpu_memory_service
Comment thread
ishandhanani marked this conversation as resolved.
Resolved
uv pip install -e '.[vllm]'
Comment thread
ishandhanani marked this conversation as resolved.
Resolved

cd ~/src/harbor
uv sync
```

First launch JIT-warms the FP8 kernels — wait for `curl localhost:8100/v1/models`
to list the model before starting the client.
These commands were validated with Harbor `0.16.0` and Pi `0.72.1`. Record the Dynamo, Harbor, and agent-plugins revisions with benchmark artifacts.

For the **KV-routing-only baseline** arm, drop the `thunderagent_router` line
from the script and run the frontend in KV-router mode against the same two
workers (`--router-mode kv`).
### 2. Launch ThunderAgent or stock KV

### 2. Run mini-SWE-agent
Choose one arm. The launcher starts both TP4 workers and the matching router, then prints readiness after both workers and the public model register.

```bash
cd ~/src/dynamo
source .venv/bin/activate
export HF_HOME=/home/nvidia/hf_cache

# ThunderAgent
export ARM=ta SESSION_FINAL=1
Comment thread
ishandhanani marked this conversation as resolved.
Resolved

# Stock KV instead
# export ARM=kv SESSION_FINAL=0

./components/src/dynamo/thunderagent_router/run_minimax_8xh100.sh "$ARM"
```

The mini-SWE-agent variant we reference is bundled in the ThunderAgent fork on
the `feat/mini-swe-direct-dynamo` branch: it patches mini-SWE-agent so that with
`MSWEA_BACKEND=dynamo` it injects Dynamo session headers natively (no proxy in the
loop), mapping each SWE-bench instance to one ThunderAgent program. Clone it,
point it at the frontend on `:8100`, and drive SWE-bench-Lite at 128 concurrent
workers:
Stop the launcher with `Ctrl-C` before starting the other arm. Always use fresh server processes for each arm.

### 3. Run one Verified task

In a second terminal, set `ARM` and `SESSION_FINAL` to match the launched stack. Use a host address reachable from Docker rather than `127.0.0.1`.

```bash
git clone -b feat/mini-swe-direct-dynamo https://github.com/ishandhanani/ThunderAgent
cd ThunderAgent/examples/inference/mini-swe-agent
# [full] pulls the swebench extras (datasets + swe-rex); a bare `-e .` can't run swebench.
uv venv && source .venv/bin/activate && uv pip install -e ".[full]"

# The bundled config defaults base_url to :8000; point it at step 1's frontend (:8100).
sed -i 's#http://localhost:8000/v1#http://localhost:8100/v1#' \
src/minisweagent/config/extra/swebench.yaml

export OPENAI_API_KEY="DUMMY" # any non-empty value; the frontend ignores it
export MSWEA_BACKEND="dynamo" # inject Dynamo session headers natively

mini-extra swebench \
--config src/minisweagent/config/extra/swebench.yaml \
--subset lite --split test --workers 128 \
--model 'MiniMaxAI/MiniMax-M2' \
--output ./swebench_out --redo-existing
cd ~/src/harbor
source .venv/bin/activate

# Match the launched arm. Use ARM=kv SESSION_FINAL=0 for stock KV.
export ARM=ta SESSION_FINAL=1
export PI_PLUGIN_DIR=~/src/agent-plugins/pi-plugin
export PYTHONPATH=$PI_PLUGIN_DIR
export DYNAMO_BASE_URL=http://$(ip route get 1.1.1.1 | awk '{print $7; exit}'):8100/v1
docker run --rm curlimages/curl:8.11.1 -fsS "$DYNAMO_BASE_URL/models"
Comment thread
ishandhanani marked this conversation as resolved.
Resolved

harbor run \
-d swebench-verified@1.0 -i astropy__astropy-12907 \
-a harbor_dynamo_pi:DynamoPi -m dynamo/MiniMaxAI/MiniMax-M2 \
--ak version=0.72.1 --ae "DYNAMO_BASE_URL=$DYNAMO_BASE_URL" \
--ae "DYN_AGENT_SESSION_FINAL=$SESSION_FINAL" \
--mounts "[{\"type\":\"bind\",\"source\":\"$PI_PLUGIN_DIR\",\"target\":\"/opt/pi-dynamo-provider\",\"read_only\":true}]" \
-n 1 --agent-setup-timeout-multiplier 10 \
--job-name "$ARM-verified-one" -y
```

For the **KV-routing-only baseline** arm, bring Dynamo up in KV-router mode
(step 1, `--router-mode kv`, no `thunderagent_router`) and rerun the same
command against the same two workers.
No pause/resume lines are expected from a one-task smoke; those only appear after the working set reaches the configured thresholds.

### 4. Run the full Verified dataset

On a fresh host, authenticate with Docker Hub and prepare every task image outside the measured interval. Verified uses 500 unique images, which exceeds the anonymous pull limit. Keeping containers until the job finishes avoids hundreds of concurrent per-trial Compose teardowns; prune them once afterward.

```bash
docker login

harbor run \
-d swebench-verified@1.0 \
-a nop \
--extra-docker-compose ~/src/dynamo/components/src/dynamo/thunderagent_router/harbor-host-network.yml \
-n 32 --n-concurrent-agents 32 \
--memory ignore \
--no-delete --environment-kwarg keep_containers=true \
--job-name verified-prepull --install-only -y

docker container prune -f
```

The host-network overlay avoids creating one Docker bridge network per trial, which exhausts Docker's default address pools during the measured run.

```bash
harbor run \
-d swebench-verified@1.0 \
-a harbor_dynamo_pi:DynamoPi -m dynamo/MiniMaxAI/MiniMax-M2 \
--ak version=0.72.1 --ae "DYNAMO_BASE_URL=$DYNAMO_BASE_URL" \
--ae "DYN_AGENT_SESSION_FINAL=$SESSION_FINAL" \
--mounts "[{\"type\":\"bind\",\"source\":\"$PI_PLUGIN_DIR\",\"target\":\"/opt/pi-dynamo-provider\",\"read_only\":true}]" \
--extra-docker-compose ~/src/dynamo/components/src/dynamo/thunderagent_router/harbor-host-network.yml \
-n 256 --n-concurrent-agents 256 \
--agent-setup-timeout-multiplier 10 --memory ignore \
--no-delete --environment-kwarg keep_containers=true \
--job-name "$ARM-verified-full" -y
```

### Expected
This runs every task in `swebench-verified@1.0` with verification enabled. Remove the stopped task containers, start the other arm from fresh processes, set its matching `ARM` and `SESSION_FINAL`, and rerun the same command.
Comment thread
ishandhanani marked this conversation as resolved.
Resolved

Over the 10–67 min steady-state window, program-aware routing
(`thunderagent_router`) sustains **≈27.5 steps/min** versus **≈23.7** for the
KV-routing-only baseline on the same two workers — an **8–16% throughput
improvement** (≈16% at the steady-state peak; ≈8.8% in the stricter matched-A/B
framing). Throughput (steps/min over the window), not resolved-rate, is the
metric at this concurrency; resolved-rate deltas are within run-to-run noise.
Stable session headers are sent in both arms. `SESSION_FINAL=0` only prevents the stock KV arm from receiving ThunderAgent's terminal lifecycle request.

## Citation

Expand All @@ -181,5 +237,5 @@ ThunderAgent paper:
- Conceptual docs: [docs/agents/thunderagent-router.md](/docs/agents/thunderagent-router.md)
- ThunderAgent paper: <https://arxiv.org/abs/2602.13692>
- Upstream ThunderAgent reference: <https://github.com/HaoKang-Timmy/ThunderAgent>
- Repro fork (mini-swe-agent + session injector): <https://github.com/ishandhanani/ThunderAgent>
- Pi Dynamo provider: <https://github.com/ai-dynamo/agent-plugins/tree/main/pi-plugin>
- Dynamo KV router: [Router Guide](/docs/components/router/router-guide.md)
36 changes: 27 additions & 9 deletions components/src/dynamo/thunderagent_router/capacity.py
Original file line number Diff line number Diff line change
@@ -1,12 +1,13 @@
# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
# SPDX-License-Identifier: Apache-2.0

"""Per-worker capacity snapshot derived from worker model deployment cards.
"""Per-worker retention budget derived from worker model deployment cards.

``capacity_tokens`` (``block_size * total_kv_blocks``) is published once per
worker via its MDC. We piggyback on ``FpmEventSubscriber`` only as the
existing Python channel that already tracks per-worker MDCs; the
forward-pass-metric payloads themselves are not consumed.
The device KV pool is always published as ``block_size * total_kv_blocks``.
When SGLang HiCache is enabled, its exact host-pool capacity is published in
runtime metadata and added to the program-retention budget. SGLang still owns
device admission, eviction, and restore; this value only prevents the program
scheduler from pausing before native GPU-to-host spill can happen.
"""

from __future__ import annotations
Expand All @@ -20,9 +21,11 @@

logger = logging.getLogger(__name__)

_SGLANG_HICACHE_CAPACITY_RUNTIME_KEY = "sglang_hicache_capacity"


class WorkerCapacityProvider:
"""Maps ``worker_id -> kv_pool_tokens`` from each worker's MDC."""
"""Maps ``worker_id -> program-retention tokens`` from each worker's MDC."""

def __init__(self, endpoint: Endpoint) -> None:
self._endpoint = endpoint
Expand Down Expand Up @@ -62,9 +65,9 @@ def snapshot(self) -> dict[int, int]:
worker_id = int(worker_id_str)
except (ValueError, TypeError):
continue
pool_tokens = self._parse_pool_tokens(card_json)
if pool_tokens is not None:
out[worker_id] = pool_tokens
retention_tokens = self._parse_pool_tokens(card_json)
if retention_tokens is not None:
out[worker_id] = retention_tokens
return out

def _parse_pool_tokens(self, card_json: str) -> Optional[int]:
Expand All @@ -85,5 +88,20 @@ def _parse_pool_tokens(self, card_json: str) -> Optional[int]:
and total_blocks > 0
):
result = int(block_size) * int(total_blocks)
runtime_data = (card.get("runtime_config") or {}).get(
"runtime_data", {}
)
hicache = (
runtime_data.get(_SGLANG_HICACHE_CAPACITY_RUNTIME_KEY, {})
if isinstance(runtime_data, dict)
else {}
)
host_tokens = (
hicache.get("host_total_tokens")
if isinstance(hicache, dict)
else None
)
if isinstance(host_tokens, (int, float)) and host_tokens > 0:
result += int(host_tokens)
Comment thread
ishandhanani marked this conversation as resolved.
Resolved
Comment thread
ishandhanani marked this conversation as resolved.
Resolved
Comment thread
ishandhanani marked this conversation as resolved.
Resolved
self._parsed[card_json] = result
return result
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
Comment thread
ishandhanani marked this conversation as resolved.
Resolved
# SPDX-License-Identifier: Apache-2.0

services:
main:
network_mode: host
Loading
Loading