diff --git a/components/src/dynamo/sglang/register.py b/components/src/dynamo/sglang/register.py index 5888bd50c945..6ee3cb913def 100644 --- a/components/src/dynamo/sglang/register.py +++ b/components/src/dynamo/sglang/register.py @@ -33,6 +33,7 @@ ) SGLANG_HICACHE_MOONCAKE_RUNTIME_KEY = "sglang_hicache_mooncake" +SGLANG_HICACHE_CAPACITY_RUNTIME_KEY = "sglang_hicache_capacity" SPEC_DECODE_RUNTIME_KEY = "spec_decode" @@ -340,6 +341,15 @@ def _eagle_enabled_for(speculative_algorithm: Optional[str]) -> bool: return False +def _get_hicache_capacity_runtime_data( + scheduler_info: dict[str, Any], +) -> Optional[dict[str, int]]: + host_total_tokens = scheduler_info.get("hicache_host_total_tokens") + if not isinstance(host_total_tokens, (int, float)) or host_total_tokens <= 0: + return None + return {"host_total_tokens": int(host_total_tokens)} + + async def _get_runtime_config( engine: sgl.Engine, server_args: ServerArgs, dynamo_args: DynamoConfig ) -> Optional[ModelRuntimeConfig]: @@ -482,12 +492,27 @@ async def _get_runtime_config( f"{unpublished} will not be published; SGLang will use its internal defaults." ) - return runtime_config - except Exception as e: logging.warning(f"Failed to get runtime config: {e}. Proceeding without it.") return runtime_config + try: + hicache_capacity_runtime_data = _get_hicache_capacity_runtime_data( + scheduler_info + ) + if hicache_capacity_runtime_data is not None: + runtime_config.set_engine_specific( + SGLANG_HICACHE_CAPACITY_RUNTIME_KEY, + json.dumps(hicache_capacity_runtime_data), + ) + logging.info("Published SGLang HiCache capacity runtime metadata.") + except Exception as e: + logging.warning( + "Failed to attach SGLang HiCache capacity runtime metadata: %s", e + ) + + return runtime_config + async def register_model_with_readiness_gate( engine: sgl.Engine, diff --git a/components/src/dynamo/sglang/tests/test_runtime_metadata.py b/components/src/dynamo/sglang/tests/test_runtime_metadata.py index 88d54e1bd9cd..19aef6bcfe01 100644 --- a/components/src/dynamo/sglang/tests/test_runtime_metadata.py +++ b/components/src/dynamo/sglang/tests/test_runtime_metadata.py @@ -70,3 +70,75 @@ def test_eagle_enabled_for_speculative_algorithm(speculative_algorithm, expected from dynamo.sglang.register import _eagle_enabled_for assert _eagle_enabled_for(speculative_algorithm) is expected + + +def test_hicache_capacity_runtime_data_uses_scheduler_capacity(): + from dynamo.sglang.register import _get_hicache_capacity_runtime_data + + assert _get_hicache_capacity_runtime_data({"hicache_host_total_tokens": 300}) == { + "host_total_tokens": 300 + } + + +@pytest.mark.parametrize("value", [None, 0, -1, "300"]) +def test_hicache_capacity_runtime_data_ignores_invalid_values(value): + from dynamo.sglang.register import _get_hicache_capacity_runtime_data + + assert ( + _get_hicache_capacity_runtime_data({"hicache_host_total_tokens": value}) is None + ) + + +@pytest.mark.asyncio +async def test_hicache_publish_failure_preserves_core_capacity(monkeypatch, caplog): + from dynamo.sglang import register + + server_args = SimpleNamespace( + context_length=4096, + disaggregation_mode=None, + max_prefill_tokens=None, + page_size=16, + speculative_algorithm="NONE", + speculative_num_steps=None, + ) + dynamo_args = register.DynamoConfig() + dynamo_args.enable_local_indexer = False + scheduler_info = { + "hicache_host_total_tokens": 300, + "max_total_num_tokens": 1024, + } + engine = SimpleNamespace( + _scheduler_init_result=SimpleNamespace(scheduler_infos=[scheduler_info]) + ) + capacity = SimpleNamespace( + max_num_seqs=None, + max_num_batched_tokens=1024, + total_kv_blocks=64, + ) + + monkeypatch.setattr(register, "model_card_dp_rank_bounds", lambda _: (0, 1)) + monkeypatch.setattr(register, "get_sglang_worker_group_id", lambda _: None) + monkeypatch.setattr( + register, "_get_bootstrap_info_for_config", lambda _: (None, None) + ) + monkeypatch.setattr(register, "_get_mooncake_runtime_data", lambda _: None) + monkeypatch.setattr(register, "runtime_capacity", lambda *_: capacity) + + original_set = register.ModelRuntimeConfig.set_engine_specific + + def fail_hicache_publish(self, key, value): + if key == register.SGLANG_HICACHE_CAPACITY_RUNTIME_KEY: + raise RuntimeError("publish failed") + return original_set(self, key, value) + + monkeypatch.setattr( + register.ModelRuntimeConfig, "set_engine_specific", fail_hicache_publish + ) + + runtime_config = await register._get_runtime_config( + engine, server_args, dynamo_args + ) + + assert runtime_config.total_kv_blocks == 64 + assert runtime_config.max_num_batched_tokens == 1024 + assert "Failed to attach SGLang HiCache capacity runtime metadata" in caplog.text diff --git a/components/src/dynamo/thunderagent_router/README.md b/components/src/dynamo/thunderagent_router/README.md index 7cf2d24f36aa..769e58772af8 100644 --- a/components/src/dynamo/thunderagent_router/README.md +++ b/components/src/dynamo/thunderagent_router/README.md @@ -11,11 +11,7 @@ requests. It wraps Dynamo's native KV router and adds a program-level scheduler with tool-boundary pause/resume, porting the scheduler from the ThunderAgent paper. -**Conceptual docs live in [docs/agents/thunderagent-router.md](/docs/agents/thunderagent-router.md)** — -the scheduler model, the 5s scheduler tick (resume → pause), tool-boundary -pause/resume semantics, the utilization-driven control loop and its full knob -table, the architecture diagram, and the scheduler observability logs. This -README keeps only the build/run/repro specifics that belong next to the code. +**Conceptual docs live in [docs/agents/thunderagent-router.md](/docs/agents/thunderagent-router.md)** — the scheduler model, tool-boundary pause/resume semantics, the utilization-driven control loop, and observability. This README contains the source build and the complete Harbor/Pi A/B walkthrough. ## Install @@ -73,6 +69,12 @@ Requests without session identity are passed through as one-off (no program admission, no pause/resume). This is the safe fallback for non-agentic traffic sharing the same workers. +### SGLang HiCache retention budget + +`dynamo.sglang` publishes the authoritative GPU KV and HiCache host capacities in each worker's model deployment card. The scheduler automatically uses their sum as its retention budget, so `--pause-threshold 0.95` means 95% of the combined GPU + host pool; there is no ThunderAgent HiCache flag to set. This lets SGLang spill from GPU to its native host tier before ThunderAgent starts holding programs at tool boundaries. + +Mooncake capacity is deliberately excluded. It is a content-addressed storage tier whose contents may be evicted or may not match the next request, not an unconditional program-retention budget. ThunderAgent does not call HiCache eviction, restore, prefetch, or Mooncake APIs; SGLang remains the admission and materialization authority. + ## Tracing Enable request tracing on the frontend with the master switch @@ -94,70 +96,127 @@ at that endpoint (producers connect) and `tool_start` / `tool_end` / `tool_call_id` pairs, giving you the full LLM-turn ↔ tool-gap timeline per agent. -## Reproducing the MiniMax-M2 results +## Harbor/Pi A/B walkthrough + +This walkthrough runs the same SWE-bench Verified task through ThunderAgent and the stock Dynamo KV router. Harbor owns the task container, Pi runs inside it, and the model stack runs on the host. ThunderAgent is backend-agnostic and works with Dynamo's vLLM and SGLang backends; this example uses one 8-GPU node and two TP4 vLLM workers loading `MiniMaxAI/MiniMax-M2.7` from Hugging Face and serving the API alias `MiniMaxAI/MiniMax-M2`. There is no HiCache, Mooncake, shared cache, or frontend admission control in either arm. -The headline numbers — program-aware scheduling vs KV-routing-only on the -same hardware — come from driving SWE-bench-Lite through **mini-SWE-agent** at -128 concurrent workers, against two TP4 MiniMax-M2 replicas on a single 8×H100 -node. +The [agent-plugins `DynamoPi` adapter](https://github.com/ai-dynamo/agent-plugins/blob/main/pi-plugin/harbor/dynamo_pi.py) is required. It installs the Dynamo provider in each Harbor task container and maps Harbor's per-trial ID to one stable `x-dynamo-session-id` across all Pi turns. The ThunderAgent arm also sends one terminal session request so the router can release the completed program; the stock KV arm disables that request because it has no lifecycle consumer. -### 1. Bring up Dynamo (2× TP4 MiniMax-M2) +### 1. Install the three source trees -One script brings up both TP4 workers, the program-aware router, and the -frontend on `:8100`: +Use Python 3.12 and a machine with Docker and eight visible GPUs. Build Dynamo from source rather than installing a released wheel. Pi and its Node.js runtime are installed inside each Harbor task container. ```bash -bash components/src/dynamo/thunderagent_router/run_minimax_8xh100.sh +git clone https://github.com/ai-dynamo/dynamo ~/src/dynamo + +git clone --branch v0.16.0 --depth 1 https://github.com/harbor-framework/harbor ~/src/harbor +git clone https://github.com/ai-dynamo/agent-plugins ~/src/agent-plugins +git -C ~/src/agent-plugins checkout 223a0b8823610d042e7479bfff9b93eeac4a23ec + +cd ~/src/dynamo +uv venv --python 3.12 --seed .venv +source .venv/bin/activate +uv pip install pip 'maturin[patchelf]' +(cd lib/bindings/python && maturin develop --uv) +uv pip install -e '.[vllm]' + +cd ~/src/harbor +uv sync ``` -First launch JIT-warms the FP8 kernels — wait for `curl localhost:8100/v1/models` -to list the model before starting the client. +These commands were validated with Harbor `0.16.0` and Pi `0.72.1`. Record the Dynamo, Harbor, and agent-plugins revisions with benchmark artifacts. + +### 2. Launch ThunderAgent or stock KV + +Choose one arm. The launcher starts both TP4 workers and the matching router, then prints readiness after both workers and the public model register. -For the **KV-routing-only baseline** arm, drop the `thunderagent_router` line -from the script and run the frontend in KV-router mode against the same two -workers (`--router-mode kv`). +```bash +cd ~/src/dynamo +source .venv/bin/activate +export HF_HOME=/home/nvidia/hf_cache + +# ThunderAgent +export ARM=ta -### 2. Run mini-SWE-agent +# Stock KV instead +# export ARM=kv -The mini-SWE-agent variant we reference is bundled in the ThunderAgent fork on -the `feat/mini-swe-direct-dynamo` branch: it patches mini-SWE-agent so that with -`MSWEA_BACKEND=dynamo` it injects Dynamo session headers natively (no proxy in the -loop), mapping each SWE-bench instance to one ThunderAgent program. Clone it, -point it at the frontend on `:8100`, and drive SWE-bench-Lite at 128 concurrent -workers: +./components/src/dynamo/thunderagent_router/run_minimax_8xh100.sh "$ARM" +``` + +Stop the launcher with `Ctrl-C` before starting the other arm. Always use fresh server processes for each arm. + +### 3. Run one Verified task + +In a second terminal, set `ARM` and `DYN_AGENT_SESSION_FINAL` to match the launched stack. Use a host address reachable from Docker rather than `127.0.0.1`. + +```bash +cd ~/src/harbor +source .venv/bin/activate + +# ThunderAgent +export ARM=ta +export DYN_AGENT_SESSION_FINAL=1 + +# Stock KV instead +# export ARM=kv +# export DYN_AGENT_SESSION_FINAL=0 + +export PI_PLUGIN_DIR=~/src/agent-plugins/pi-plugin +export PYTHONPATH=$PI_PLUGIN_DIR/harbor +export DYNAMO_BASE_URL=http://$(ip route get 1.1.1.1 | awk '{print $7; exit}'):8100/v1 +curl -fsS "$DYNAMO_BASE_URL/models" + +harbor run \ + -d swebench-verified@1.0 -i astropy__astropy-12907 \ + -a dynamo_pi:DynamoPi -m dynamo/MiniMaxAI/MiniMax-M2 \ + --ak version=0.72.1 --ae "DYNAMO_BASE_URL=$DYNAMO_BASE_URL" \ + --ae "DYN_AGENT_SESSION_FINAL=$DYN_AGENT_SESSION_FINAL" \ + --mounts "[{\"type\":\"bind\",\"source\":\"$PI_PLUGIN_DIR\",\"target\":\"/opt/pi-dynamo-provider\",\"read_only\":true}]" \ + -n 1 --agent-setup-timeout-multiplier 10 \ + --job-name "$ARM-verified-one" -y +``` + +No pause/resume lines are expected from a one-task smoke; those only appear after the working set reaches the configured thresholds. + +### 4. Run the full Verified dataset + +On a fresh host, authenticate with Docker Hub and prepare every task image outside the measured interval. Verified uses 500 unique images, which exceeds the anonymous pull limit. Keeping containers until the job finishes avoids hundreds of concurrent per-trial Compose teardowns; prune them once afterward. ```bash -git clone -b feat/mini-swe-direct-dynamo https://github.com/ishandhanani/ThunderAgent -cd ThunderAgent/examples/inference/mini-swe-agent -# [full] pulls the swebench extras (datasets + swe-rex); a bare `-e .` can't run swebench. -uv venv && source .venv/bin/activate && uv pip install -e ".[full]" - -# The bundled config defaults base_url to :8000; point it at step 1's frontend (:8100). -sed -i 's#http://localhost:8000/v1#http://localhost:8100/v1#' \ - src/minisweagent/config/extra/swebench.yaml - -export OPENAI_API_KEY="DUMMY" # any non-empty value; the frontend ignores it -export MSWEA_BACKEND="dynamo" # inject Dynamo session headers natively - -mini-extra swebench \ - --config src/minisweagent/config/extra/swebench.yaml \ - --subset lite --split test --workers 128 \ - --model 'MiniMaxAI/MiniMax-M2' \ - --output ./swebench_out --redo-existing +docker login + +harbor run \ + -d swebench-verified@1.0 \ + -a nop \ + --extra-docker-compose ~/src/agent-plugins/pi-plugin/harbor/host-network.yml \ + -n 32 --n-concurrent-agents 32 \ + --memory ignore \ + --no-delete --environment-kwarg keep_containers=true \ + --job-name verified-prepull --install-only -y + +docker container prune -f ``` -For the **KV-routing-only baseline** arm, bring Dynamo up in KV-router mode -(step 1, `--router-mode kv`, no `thunderagent_router`) and rerun the same -command against the same two workers. +The host-network overlay avoids creating one Docker bridge network per trial, which exhausts Docker's default address pools during the measured run. + +```bash +harbor run \ + -d swebench-verified@1.0 \ + -a dynamo_pi:DynamoPi -m dynamo/MiniMaxAI/MiniMax-M2 \ + --ak version=0.72.1 --ae "DYNAMO_BASE_URL=$DYNAMO_BASE_URL" \ + --ae "DYN_AGENT_SESSION_FINAL=$DYN_AGENT_SESSION_FINAL" \ + --mounts "[{\"type\":\"bind\",\"source\":\"$PI_PLUGIN_DIR\",\"target\":\"/opt/pi-dynamo-provider\",\"read_only\":true}]" \ + --extra-docker-compose ~/src/agent-plugins/pi-plugin/harbor/host-network.yml \ + -n 256 --n-concurrent-agents 256 \ + --agent-setup-timeout-multiplier 10 --memory ignore \ + --no-delete --environment-kwarg keep_containers=true \ + --job-name "$ARM-verified-full" -y +``` -### Expected +This runs every task in `swebench-verified@1.0` with verification enabled. Remove the stopped task containers, start the other arm from fresh processes, set its matching `ARM` and `DYN_AGENT_SESSION_FINAL`, and rerun the same command. -Over the 10–67 min steady-state window, program-aware routing -(`thunderagent_router`) sustains **≈27.5 steps/min** versus **≈23.7** for the -KV-routing-only baseline on the same two workers — an **8–16% throughput -improvement** (≈16% at the steady-state peak; ≈8.8% in the stricter matched-A/B -framing). Throughput (steps/min over the window), not resolved-rate, is the -metric at this concurrency; resolved-rate deltas are within run-to-run noise. +Stable session headers are sent in both arms. `DYN_AGENT_SESSION_FINAL=0` only prevents the stock KV arm from receiving ThunderAgent's terminal lifecycle request. ## Citation @@ -181,5 +240,5 @@ ThunderAgent paper: - Conceptual docs: [docs/agents/thunderagent-router.md](/docs/agents/thunderagent-router.md) - ThunderAgent paper: - Upstream ThunderAgent reference: -- Repro fork (mini-swe-agent + session injector): +- Pi Dynamo provider: - Dynamo KV router: [Router Guide](/docs/components/router/router-guide.md) diff --git a/components/src/dynamo/thunderagent_router/capacity.py b/components/src/dynamo/thunderagent_router/capacity.py index 1d31bae80ab1..e6219bd31b0a 100644 --- a/components/src/dynamo/thunderagent_router/capacity.py +++ b/components/src/dynamo/thunderagent_router/capacity.py @@ -1,12 +1,13 @@ # SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 -"""Per-worker capacity snapshot derived from worker model deployment cards. +"""Per-worker retention budget derived from worker model deployment cards. -``capacity_tokens`` (``block_size * total_kv_blocks``) is published once per -worker via its MDC. We piggyback on ``FpmEventSubscriber`` only as the -existing Python channel that already tracks per-worker MDCs; the -forward-pass-metric payloads themselves are not consumed. +The device KV pool is always published as ``block_size * total_kv_blocks``. +When SGLang HiCache is enabled, its exact host-pool capacity is published in +runtime metadata and added to the program-retention budget. SGLang still owns +device admission, eviction, and restore; this value only prevents the program +scheduler from pausing before native GPU-to-host spill can happen. """ from __future__ import annotations @@ -20,9 +21,11 @@ logger = logging.getLogger(__name__) +_SGLANG_HICACHE_CAPACITY_RUNTIME_KEY = "sglang_hicache_capacity" + class WorkerCapacityProvider: - """Maps ``worker_id -> kv_pool_tokens`` from each worker's MDC.""" + """Maps ``worker_id -> program-retention tokens`` from each worker's MDC.""" def __init__(self, endpoint: Endpoint) -> None: self._endpoint = endpoint @@ -62,9 +65,9 @@ def snapshot(self) -> dict[int, int]: worker_id = int(worker_id_str) except (ValueError, TypeError): continue - pool_tokens = self._parse_pool_tokens(card_json) - if pool_tokens is not None: - out[worker_id] = pool_tokens + retention_tokens = self._parse_pool_tokens(card_json) + if retention_tokens is not None: + out[worker_id] = retention_tokens return out def _parse_pool_tokens(self, card_json: str) -> Optional[int]: @@ -85,5 +88,20 @@ def _parse_pool_tokens(self, card_json: str) -> Optional[int]: and total_blocks > 0 ): result = int(block_size) * int(total_blocks) + runtime_data = (card.get("runtime_config") or {}).get( + "runtime_data", {} + ) + hicache = ( + runtime_data.get(_SGLANG_HICACHE_CAPACITY_RUNTIME_KEY, {}) + if isinstance(runtime_data, dict) + else {} + ) + host_tokens = ( + hicache.get("host_total_tokens") + if isinstance(hicache, dict) + else None + ) + if isinstance(host_tokens, (int, float)) and host_tokens > 0: + result += int(host_tokens) self._parsed[card_json] = result return result diff --git a/components/src/dynamo/thunderagent_router/run_minimax_8xh100.sh b/components/src/dynamo/thunderagent_router/run_minimax_8xh100.sh index 10b0e90a36a5..6e73746040bf 100755 --- a/components/src/dynamo/thunderagent_router/run_minimax_8xh100.sh +++ b/components/src/dynamo/thunderagent_router/run_minimax_8xh100.sh @@ -2,46 +2,89 @@ # SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 # -# Bring up the thunderagent_router MiniMax-M2 eval stack on a single 8xH100 -# node: two TP4 vLLM workers (GPUs 0-3 and 4-7) + the program-aware router + -# the frontend on :8100. Hardcoded for that one config -- nothing else. -# -# Pair it with the mini-SWE-agent client (see README "Reproducing the MiniMax-M2 -# results"). First launch JIT-warms the FP8 kernels and can take several -# minutes before /v1/models responds. +# Bring up the MiniMax-M2 A/B stack on one 8xH100 node. Usage: +# ./run_minimax_8xh100.sh ta +# ./run_minimax_8xh100.sh kv set -euo pipefail -trap 'echo "Cleaning up..."; kill 0' EXIT -MODEL="MiniMaxAI/MiniMax-M2" +SCRIPT_DIR="$(dirname "$(readlink -f "$0")")" +source "$SCRIPT_DIR/../../../../examples/common/launch_utils.sh" + +POLICY="${1:-}" +if [[ "$POLICY" != "ta" && "$POLICY" != "kv" ]]; then + echo "usage: $0 ta|kv" >&2 + exit 2 +fi + +trap dynamo_exit_trap EXIT + +MODEL_PATH="${MODEL_PATH:-MiniMaxAI/MiniMax-M2.7}" +MODEL_NAME_ROUTER="${MODEL_NAME_ROUTER:-MiniMaxAI/MiniMax-M2}" +WORKER_MODEL="$MODEL_NAME_ROUTER" BLOCK_SIZE=16 HTTP_PORT=8100 -export PYTHONHASHSEED=0 # deterministic KV-event block hashes +if [[ "$POLICY" == "ta" ]]; then + WORKER_MODEL="dyn-internal-minimax-m2" +fi -# Worker 0: GPUs 0-3. -DYN_SYSTEM_PORT=8081 CUDA_VISIBLE_DEVICES=0,1,2,3 python -m dynamo.vllm \ - --model "$MODEL" --tensor-parallel-size 4 --block-size "$BLOCK_SIZE" \ +export PYTHONHASHSEED=0 +export DYN_DISCOVERY_BACKEND=file +export DYN_FILE_KV="${DYN_FILE_KV:-/tmp/dynamo-minimax-${POLICY}-$$}" +export DYN_REQUEST_PLANE=tcp +export DYN_EVENT_PLANE=zmq +mkdir -p "$DYN_FILE_KV" + +DYN_SYSTEM_PORT=8181 DYN_FORWARDPASS_METRIC_PORT=20081 \ +VLLM_NIXL_SIDE_CHANNEL_PORT=20097 CUDA_VISIBLE_DEVICES=0,1,2,3 \ +python -m dynamo.vllm \ + --model "$MODEL_PATH" --served-model-name "$WORKER_MODEL" \ + --tensor-parallel-size 4 --block-size "$BLOCK_SIZE" \ + --kv-cache-dtype fp8 --enable-prefix-caching \ --dyn-tool-call-parser minimax_m2 \ + --dyn-reasoning-parser minimax_append_think \ --kv-events-config '{"publisher":"zmq","topic":"kv-events","endpoint":"tcp://*:20080","enable_kv_cache_events":true}' & -# Worker 1: GPUs 4-7. -DYN_SYSTEM_PORT=8082 CUDA_VISIBLE_DEVICES=4,5,6,7 python -m dynamo.vllm \ - --model "$MODEL" --tensor-parallel-size 4 --block-size "$BLOCK_SIZE" \ +DYN_SYSTEM_PORT=8182 DYN_FORWARDPASS_METRIC_PORT=20082 \ +VLLM_NIXL_SIDE_CHANNEL_PORT=20098 CUDA_VISIBLE_DEVICES=4,5,6,7 \ +python -m dynamo.vllm \ + --model "$MODEL_PATH" --served-model-name "$WORKER_MODEL" \ + --tensor-parallel-size 4 --block-size "$BLOCK_SIZE" \ + --kv-cache-dtype fp8 --enable-prefix-caching \ --dyn-tool-call-parser minimax_m2 \ - --kv-events-config '{"publisher":"zmq","topic":"kv-events","endpoint":"tcp://*:20081","enable_kv_cache_events":true}' & + --dyn-reasoning-parser minimax_append_think \ + --kv-events-config '{"publisher":"zmq","topic":"kv-events","endpoint":"tcp://*:20090","enable_kv_cache_events":true}' & -# Program-aware router: registers the model handler and forwards the parser so -# MiniMax's XML reaches the agent as OpenAI tool_calls. -python -m dynamo.thunderagent_router \ - --endpoint dynamo.vllm.generate \ - --model-name "$MODEL" \ - --dyn-tool-call-parser minimax_m2 \ - --router-block-size "$BLOCK_SIZE" & +if [[ "$POLICY" == "ta" ]]; then + DYN_SYSTEM_PORT=8183 python -m dynamo.thunderagent_router \ + --endpoint dynamo.backend.generate \ + --model-name "$MODEL_NAME_ROUTER" \ + --model-path "$MODEL_PATH" \ + --dyn-tool-call-parser minimax_m2 \ + --dyn-reasoning-parser minimax_append_think \ + --router-block-size "$BLOCK_SIZE" \ + --router-reset-states \ + --shared-cache-type none & + ROUTER_MODE=round-robin +else + ROUTER_MODE=kv +fi -# Frontend (round-robin; the router owns scheduling and registered the model). -python -m dynamo.frontend \ +DYN_SYSTEM_PORT=8184 python -m dynamo.frontend \ + --http-host 0.0.0.0 \ --http-port "$HTTP_PORT" \ - --router-mode round-robin \ - --router-reset-states & + --router-mode "$ROUTER_MODE" \ + --router-reset-states \ + --shared-cache-type none \ + --admission-control none & + +until curl -fsS "http://127.0.0.1:${HTTP_PORT}/v1/models/${WORKER_MODEL}/ready" 2>/dev/null \ + | jq -e '([.namespaces[].worker_types.aggregated.workers // 0] | add) == 2' >/dev/null; do + sleep 5 +done +until curl -fsS "http://127.0.0.1:${HTTP_PORT}/v1/models" 2>/dev/null | grep -Fq "$MODEL_NAME_ROUTER"; do + sleep 5 +done +echo "$POLICY stack ready at http://127.0.0.1:${HTTP_PORT}/v1" -wait +wait_any_exit diff --git a/components/src/dynamo/thunderagent_router/tests/test_capacity.py b/components/src/dynamo/thunderagent_router/tests/test_capacity.py index 67f5a4364e0d..3f1b01dce020 100644 --- a/components/src/dynamo/thunderagent_router/tests/test_capacity.py +++ b/components/src/dynamo/thunderagent_router/tests/test_capacity.py @@ -34,12 +34,20 @@ def _make_provider( return provider, subscriber -def _card(block_size: Optional[int], total_blocks: Optional[int]) -> str: +def _card( + block_size: Optional[int], + total_blocks: Optional[int], + host_total_tokens: Optional[int] = None, +) -> str: body: dict = {} if block_size is not None: body["kv_cache_block_size"] = block_size if total_blocks is not None: body["runtime_config"] = {"total_kv_blocks": total_blocks} + if host_total_tokens is not None: + body.setdefault("runtime_config", {}).setdefault("runtime_data", {})[ + "sglang_hicache_capacity" + ] = {"host_total_tokens": host_total_tokens} return json.dumps(body) @@ -48,6 +56,11 @@ def test_snapshot_extracts_kv_pool_tokens(): assert provider.snapshot() == {1: 16_000, 2: 16_000} +def test_snapshot_adds_hicache_host_tokens_to_retention_budget(): + provider, _ = _make_provider({"1": _card(16, 1_000, host_total_tokens=300)}) + assert provider.snapshot() == {1: 16_300} + + def test_snapshot_skips_malformed_cards(): provider, _ = _make_provider( { diff --git a/docs/agents/agent-harnesses.md b/docs/agents/agent-harnesses.md index cfa9ad6a6673..333248efeada 100644 --- a/docs/agents/agent-harnesses.md +++ b/docs/agents/agent-harnesses.md @@ -35,6 +35,26 @@ codex -m zai-org/GLM-4.7-Flash -c model_provider=dynamo Codex sends a `session-id` header that Dynamo maps to `session_id`. +## Pi + +Pi uses the Dynamo provider plugin. Build and install it from the [agent-plugins](https://github.com/ai-dynamo/agent-plugins/tree/main/pi-plugin) checkout: + +```bash +git clone https://github.com/ai-dynamo/agent-plugins.git ~/agent-plugins +cd ~/agent-plugins/pi-plugin +npm install && npm run build +pi install "$PWD" +``` + +Point it at the Dynamo OpenAI-compatible endpoint and run Pi with the `dynamo` provider: + +```bash +export DYNAMO_BASE_URL=http://localhost:8000/v1 +export DYNAMO_API_KEY=dynamo-local + +pi --model dynamo/zai-org/GLM-4.7-Flash +``` + ## Claude Code Claude Code uses Anthropic-compatible Messages API. The local launcher above starts `dynamo.frontend` with `--enable-anthropic-api`; for other deployments, pass that flag when starting the frontend. Then set: diff --git a/docs/agents/thunderagent-router.md b/docs/agents/thunderagent-router.md index 5b3ed562d83e..3148ab76d4c1 100644 --- a/docs/agents/thunderagent-router.md +++ b/docs/agents/thunderagent-router.md @@ -5,7 +5,7 @@ title: ThunderAgent Program Scheduler subtitle: Program-level scheduling with tool-boundary pause/resume on top of KV-aware routing --- -> **Experimental — not a released component.** Run it from a source checkout, not from a `pip install ai-dynamo`. The CLI flags, session headers, and lifecycle hooks are all unstable and will change. Build and launch specifics live next to the code in [`components/src/dynamo/thunderagent_router/README.md`](../../components/src/dynamo/thunderagent_router/README.md). +> **Experimental — not a released component.** Run it from a source checkout, not from a `pip install ai-dynamo`. The CLI flags, session headers, and lifecycle hooks are all unstable and will change. `dynamo.thunderagent_router` is a standalone Dynamo router that schedules at the granularity of an agent run — the whole `LLM turn → tool call → next turn` loop — instead of individual requests. It wraps Dynamo's native KV router and adds a program-level scheduler with tool-boundary pause/resume on top of KV-aware routing, porting the scheduler from the [ThunderAgent](https://arxiv.org/abs/2602.13692) paper (Kang et al., 2026). @@ -43,7 +43,7 @@ A program is created on its first turn, keyed by `session_id`. Public session id ## Utilization-Driven Control Loop -Pause/resume is driven by per-worker utilization — the program working set as a fraction of the worker's KV pool. The loop has three bands: +Pause/resume is driven by per-worker utilization — the program working set as a fraction of the worker's retention budget. With SGLang HiCache enabled, Dynamo reads the worker's published GPU KV and host HiCache capacities and uses their sum. The host tier is included so native GPU-to-host spill can happen before this scheduler pauses programs. Mooncake is excluded: it is conditional content-addressed storage, not guaranteed per-program retention. The loop has three bands: - At or above `pause-threshold`, the worker is over-subscribed; the tick pauses ACTING programs until utilization falls back to `pause-target`. - In the `[soft-demote-threshold, pause-threshold)` band, programs are soft-demoted (a negative priority jump) but not paused — early backpressure before a hard pause is needed. @@ -51,7 +51,7 @@ Pause/resume is driven by per-worker utilization — the program working set as | Flag | Env var | Default | Description | |---|---|---|---| -| `--pause-threshold` | `DYN_THUNDERAGENT_PAUSE_THRESHOLD` | `0.95` | Working-set fraction of the KV pool that fires a pause cycle. | +| `--pause-threshold` | `DYN_THUNDERAGENT_PAUSE_THRESHOLD` | `0.95` | Working-set fraction of the retention budget that fires a pause cycle. | | `--soft-demote-threshold` | `DYN_THUNDERAGENT_SOFT_DEMOTE_THRESHOLD` | `0.80` | Soft-demote band start (negative priority jump in `[soft, pause)`). | | `--pause-target` | `DYN_THUNDERAGENT_PAUSE_TARGET` | `0.80` | Setpoint that pause cycles drive utilization back down to. Must be `<= pause-threshold`. | | `--resume-hysteresis` | `DYN_THUNDERAGENT_RESUME_HYSTERESIS` | `0.10` | Headroom below `pause-threshold` required before any resume. | @@ -64,7 +64,7 @@ Pause/resume is driven by per-worker utilization — the program working set as > **Constraint:** `pause-target <= pause-threshold`. The service rejects configs that violate it (along with `0 <= resume-hysteresis <= pause-threshold` and `0 <= soft-demote-threshold <= pause-threshold`). -All `KvRouter` flags from `dynamo.router` (`--router-temperature`, `--use-kv-events`, `--router-track-output-blocks`, …) are also accepted and forwarded. See the [folder README](../../components/src/dynamo/thunderagent_router/README.md) for the remaining service flags (`--endpoint`, `--model-name`, `--model-path`, tool-call and reasoning parsers). +All `KvRouter` flags from `dynamo.router` (`--router-temperature`, `--use-kv-events`, `--router-track-output-blocks`, …) are also accepted and forwarded. ## Architecture @@ -126,14 +126,14 @@ Enable these by lowering the log level for `dynamo.thunderagent_router`. They gi For per-request tracing (token counts, cache hits, worker placement), the router also integrates with [Agent Tracing](agent-tracing.md#enable-output): set `DYN_REQUEST_TRACE=1` on the frontend to land a `request_end` record per LLM call. Harness tool-event spans are separate: they require `DYN_REQUEST_TRACE_TOOL_EVENTS_ZMQ_ENDPOINT` plus a configured publisher. -## Reproducing the MiniMax-M2 Results +## Reproducing with upstream Harbor and Pi -The headline numbers (program-aware scheduling vs KV-routing-only on the same hardware, ~12-16% throughput improvement on SWE-bench-Lite with two TP4 MiniMax-M2 replicas on a single 8×H100 node) and the exact launch/repro commands live in the [folder README](../../components/src/dynamo/thunderagent_router/README.md). +The maintained end-to-end path uses upstream Harbor to create SWE-bench containers and Pi with the Dynamo provider inside each container. The complete source build, ThunderAgent arm, stock KV arm, stable-session setup, and scaling procedure live in the [`thunderagent_router` README](https://github.com/ai-dynamo/dynamo/blob/main/components/src/dynamo/thunderagent_router/README.md#harborpi-ab-walkthrough). ## References - ThunderAgent paper: [arxiv.org/abs/2602.13692](https://arxiv.org/abs/2602.13692) - Upstream ThunderAgent reference: [HaoKang-Timmy/ThunderAgent](https://github.com/HaoKang-Timmy/ThunderAgent) -- Repro fork (mini-swe-agent + session injector): [ishandhanani/ThunderAgent](https://github.com/ishandhanani/ThunderAgent) +- Pi Dynamo provider: [ai-dynamo/agent-plugins](https://github.com/ai-dynamo/agent-plugins/tree/main/pi-plugin) - Dynamo KV router: [Router Guide](../components/router/router-guide.md) - [Session IDs](session-ids.md), [Agent Tracing](agent-tracing.md), and [Agent Hints](agent-hints.md)