From 398c54343da6d22d25caaeb5b1bd0a4fe65db786 Mon Sep 17 00:00:00 2001 From: Ishan Dhanani Date: Wed, 1 Jul 2026 21:54:06 +0000 Subject: [PATCH 01/30] feat(thunderagent): include hicache retention budget Signed-off-by: Ishan Dhanani --- components/src/dynamo/sglang/register.py | 18 ++++++++++ .../sglang/tests/test_runtime_metadata.py | 15 ++++++++ .../dynamo/thunderagent_router/capacity.py | 36 ++++++++++++++----- .../tests/test_capacity.py | 15 +++++++- 4 files changed, 74 insertions(+), 10 deletions(-) diff --git a/components/src/dynamo/sglang/register.py b/components/src/dynamo/sglang/register.py index 5888bd50c945..eae6c10b9712 100644 --- a/components/src/dynamo/sglang/register.py +++ b/components/src/dynamo/sglang/register.py @@ -33,6 +33,7 @@ ) SGLANG_HICACHE_MOONCAKE_RUNTIME_KEY = "sglang_hicache_mooncake" +SGLANG_HICACHE_CAPACITY_RUNTIME_KEY = "sglang_hicache_capacity" SPEC_DECODE_RUNTIME_KEY = "spec_decode" @@ -340,6 +341,15 @@ def _eagle_enabled_for(speculative_algorithm: Optional[str]) -> bool: return False +def _get_hicache_capacity_runtime_data( + scheduler_info: dict[str, Any], +) -> Optional[dict[str, int]]: + host_total_tokens = scheduler_info.get("hicache_host_total_tokens") + if not isinstance(host_total_tokens, (int, float)) or host_total_tokens <= 0: + return None + return {"host_total_tokens": int(host_total_tokens)} + + async def _get_runtime_config( engine: sgl.Engine, server_args: ServerArgs, dynamo_args: DynamoConfig ) -> Optional[ModelRuntimeConfig]: @@ -454,6 +464,14 @@ async def _get_runtime_config( try: scheduler_info = engine._scheduler_init_result.scheduler_infos[0] + hicache_capacity_runtime_data = _get_hicache_capacity_runtime_data( + scheduler_info + ) + if hicache_capacity_runtime_data is not None: + runtime_config.set_engine_specific( + SGLANG_HICACHE_CAPACITY_RUNTIME_KEY, + json.dumps(hicache_capacity_runtime_data), + ) capacity = runtime_capacity(server_args, scheduler_info) max_total_tokens = scheduler_info.get("max_total_num_tokens") diff --git a/components/src/dynamo/sglang/tests/test_runtime_metadata.py b/components/src/dynamo/sglang/tests/test_runtime_metadata.py index 88d54e1bd9cd..8f7f54b2aa53 100644 --- a/components/src/dynamo/sglang/tests/test_runtime_metadata.py +++ b/components/src/dynamo/sglang/tests/test_runtime_metadata.py @@ -6,6 +6,7 @@ import pytest from dynamo.sglang.capacity import get_spec_decode_runtime_data +from dynamo.sglang.register import _get_hicache_capacity_runtime_data pytestmark = [ pytest.mark.unit, @@ -70,3 +71,17 @@ def test_eagle_enabled_for_speculative_algorithm(speculative_algorithm, expected from dynamo.sglang.register import _eagle_enabled_for assert _eagle_enabled_for(speculative_algorithm) is expected + + +def test_hicache_capacity_runtime_data_uses_scheduler_capacity(): + assert _get_hicache_capacity_runtime_data({"hicache_host_total_tokens": 300}) == { + "host_total_tokens": 300 + } + + +@pytest.mark.parametrize("value", [None, 0, -1, "300"]) +def test_hicache_capacity_runtime_data_ignores_invalid_values(value): + assert ( + _get_hicache_capacity_runtime_data({"hicache_host_total_tokens": value}) + is None + ) diff --git a/components/src/dynamo/thunderagent_router/capacity.py b/components/src/dynamo/thunderagent_router/capacity.py index 1d31bae80ab1..e6219bd31b0a 100644 --- a/components/src/dynamo/thunderagent_router/capacity.py +++ b/components/src/dynamo/thunderagent_router/capacity.py @@ -1,12 +1,13 @@ # SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 -"""Per-worker capacity snapshot derived from worker model deployment cards. +"""Per-worker retention budget derived from worker model deployment cards. -``capacity_tokens`` (``block_size * total_kv_blocks``) is published once per -worker via its MDC. We piggyback on ``FpmEventSubscriber`` only as the -existing Python channel that already tracks per-worker MDCs; the -forward-pass-metric payloads themselves are not consumed. +The device KV pool is always published as ``block_size * total_kv_blocks``. +When SGLang HiCache is enabled, its exact host-pool capacity is published in +runtime metadata and added to the program-retention budget. SGLang still owns +device admission, eviction, and restore; this value only prevents the program +scheduler from pausing before native GPU-to-host spill can happen. """ from __future__ import annotations @@ -20,9 +21,11 @@ logger = logging.getLogger(__name__) +_SGLANG_HICACHE_CAPACITY_RUNTIME_KEY = "sglang_hicache_capacity" + class WorkerCapacityProvider: - """Maps ``worker_id -> kv_pool_tokens`` from each worker's MDC.""" + """Maps ``worker_id -> program-retention tokens`` from each worker's MDC.""" def __init__(self, endpoint: Endpoint) -> None: self._endpoint = endpoint @@ -62,9 +65,9 @@ def snapshot(self) -> dict[int, int]: worker_id = int(worker_id_str) except (ValueError, TypeError): continue - pool_tokens = self._parse_pool_tokens(card_json) - if pool_tokens is not None: - out[worker_id] = pool_tokens + retention_tokens = self._parse_pool_tokens(card_json) + if retention_tokens is not None: + out[worker_id] = retention_tokens return out def _parse_pool_tokens(self, card_json: str) -> Optional[int]: @@ -85,5 +88,20 @@ def _parse_pool_tokens(self, card_json: str) -> Optional[int]: and total_blocks > 0 ): result = int(block_size) * int(total_blocks) + runtime_data = (card.get("runtime_config") or {}).get( + "runtime_data", {} + ) + hicache = ( + runtime_data.get(_SGLANG_HICACHE_CAPACITY_RUNTIME_KEY, {}) + if isinstance(runtime_data, dict) + else {} + ) + host_tokens = ( + hicache.get("host_total_tokens") + if isinstance(hicache, dict) + else None + ) + if isinstance(host_tokens, (int, float)) and host_tokens > 0: + result += int(host_tokens) self._parsed[card_json] = result return result diff --git a/components/src/dynamo/thunderagent_router/tests/test_capacity.py b/components/src/dynamo/thunderagent_router/tests/test_capacity.py index 67f5a4364e0d..3f1b01dce020 100644 --- a/components/src/dynamo/thunderagent_router/tests/test_capacity.py +++ b/components/src/dynamo/thunderagent_router/tests/test_capacity.py @@ -34,12 +34,20 @@ def _make_provider( return provider, subscriber -def _card(block_size: Optional[int], total_blocks: Optional[int]) -> str: +def _card( + block_size: Optional[int], + total_blocks: Optional[int], + host_total_tokens: Optional[int] = None, +) -> str: body: dict = {} if block_size is not None: body["kv_cache_block_size"] = block_size if total_blocks is not None: body["runtime_config"] = {"total_kv_blocks": total_blocks} + if host_total_tokens is not None: + body.setdefault("runtime_config", {}).setdefault("runtime_data", {})[ + "sglang_hicache_capacity" + ] = {"host_total_tokens": host_total_tokens} return json.dumps(body) @@ -48,6 +56,11 @@ def test_snapshot_extracts_kv_pool_tokens(): assert provider.snapshot() == {1: 16_000, 2: 16_000} +def test_snapshot_adds_hicache_host_tokens_to_retention_budget(): + provider, _ = _make_provider({"1": _card(16, 1_000, host_total_tokens=300)}) + assert provider.snapshot() == {1: 16_300} + + def test_snapshot_skips_malformed_cards(): provider, _ = _make_provider( { From d4e6fcbe125d71286a812b7c94e0563617ee6d16 Mon Sep 17 00:00:00 2001 From: Ishan Dhanani Date: Thu, 2 Jul 2026 20:04:51 +0000 Subject: [PATCH 02/30] docs(agents): update ThunderAgent HiCache and Pi repro Signed-off-by: Ishan Dhanani --- .../src/dynamo/thunderagent_router/README.md | 77 +++++-------------- docs/agents/agent-harnesses.md | 23 ++++++ docs/agents/thunderagent-router.md | 10 +-- 3 files changed, 49 insertions(+), 61 deletions(-) diff --git a/components/src/dynamo/thunderagent_router/README.md b/components/src/dynamo/thunderagent_router/README.md index 7cf2d24f36aa..ec472be1821a 100644 --- a/components/src/dynamo/thunderagent_router/README.md +++ b/components/src/dynamo/thunderagent_router/README.md @@ -73,6 +73,12 @@ Requests without session identity are passed through as one-off (no program admission, no pause/resume). This is the safe fallback for non-agentic traffic sharing the same workers. +### SGLang HiCache retention budget + +`dynamo.sglang` publishes the authoritative GPU KV and HiCache host capacities in each worker's model deployment card. The scheduler automatically uses their sum as its retention budget, so `--pause-threshold 0.95` means 95% of the combined GPU + host pool; there is no ThunderAgent HiCache flag to set. This lets SGLang spill from GPU to its native host tier before ThunderAgent starts holding programs at tool boundaries. + +Mooncake capacity is deliberately excluded. It is a content-addressed storage tier whose contents may be evicted or may not match the next request, not an unconditional program-retention budget. ThunderAgent does not call HiCache eviction, restore, prefetch, or Mooncake APIs; SGLang remains the admission and materialization authority. + ## Tracing Enable request tracing on the frontend with the master switch @@ -94,70 +100,29 @@ at that endpoint (producers connect) and `tool_start` / `tool_end` / `tool_call_id` pairs, giving you the full LLM-turn ↔ tool-gap timeline per agent. -## Reproducing the MiniMax-M2 results - -The headline numbers — program-aware scheduling vs KV-routing-only on the -same hardware — come from driving SWE-bench-Lite through **mini-SWE-agent** at -128 concurrent workers, against two TP4 MiniMax-M2 replicas on a single 8×H100 -node. - -### 1. Bring up Dynamo (2× TP4 MiniMax-M2) +## Reproducing with Pi -One script brings up both TP4 workers, the program-aware router, and the -frontend on `:8100`: +The maintained smoke path is Pi through the Dynamo provider, not the old private mini-SWE-agent fork. Start the workers, this router, and the frontend as described above, then wait for the frontend to advertise the served model: ```bash -bash components/src/dynamo/thunderagent_router/run_minimax_8xh100.sh +curl -fsS http://127.0.0.1:8000/v1/models ``` -First launch JIT-warms the FP8 kernels — wait for `curl localhost:8100/v1/models` -to list the model before starting the client. - -For the **KV-routing-only baseline** arm, drop the `thunderagent_router` line -from the script and run the frontend in KV-router mode against the same two -workers (`--router-mode kv`). - -### 2. Run mini-SWE-agent - -The mini-SWE-agent variant we reference is bundled in the ThunderAgent fork on -the `feat/mini-swe-direct-dynamo` branch: it patches mini-SWE-agent so that with -`MSWEA_BACKEND=dynamo` it injects Dynamo session headers natively (no proxy in the -loop), mapping each SWE-bench instance to one ThunderAgent program. Clone it, -point it at the frontend on `:8100`, and drive SWE-bench-Lite at 128 concurrent -workers: +Install the [Pi Dynamo provider](https://github.com/ai-dynamo/agent-plugins/tree/main/pi-plugin) from a checkout, then send a sessionized turn through the frontend: ```bash -git clone -b feat/mini-swe-direct-dynamo https://github.com/ishandhanani/ThunderAgent -cd ThunderAgent/examples/inference/mini-swe-agent -# [full] pulls the swebench extras (datasets + swe-rex); a bare `-e .` can't run swebench. -uv venv && source .venv/bin/activate && uv pip install -e ".[full]" - -# The bundled config defaults base_url to :8000; point it at step 1's frontend (:8100). -sed -i 's#http://localhost:8000/v1#http://localhost:8100/v1#' \ - src/minisweagent/config/extra/swebench.yaml - -export OPENAI_API_KEY="DUMMY" # any non-empty value; the frontend ignores it -export MSWEA_BACKEND="dynamo" # inject Dynamo session headers natively - -mini-extra swebench \ - --config src/minisweagent/config/extra/swebench.yaml \ - --subset lite --split test --workers 128 \ - --model 'MiniMaxAI/MiniMax-M2' \ - --output ./swebench_out --redo-existing +git clone https://github.com/ai-dynamo/agent-plugins.git +cd agent-plugins/pi-plugin +npm install && npm run build +pi install "$PWD" + +export DYNAMO_BASE_URL=http://127.0.0.1:8000/v1 +export DYNAMO_API_KEY=dummy +export DYN_AGENT_SESSION_ID=ta-repro-$(uuidgen) +pi --model dynamo/ -p 'Reply exactly READY.' ``` -For the **KV-routing-only baseline** arm, bring Dynamo up in KV-router mode -(step 1, `--router-mode kv`, no `thunderagent_router`) and rerun the same -command against the same two workers. - -### Expected - -Over the 10–67 min steady-state window, program-aware routing -(`thunderagent_router`) sustains **≈27.5 steps/min** versus **≈23.7** for the -KV-routing-only baseline on the same two workers — an **8–16% throughput -improvement** (≈16% at the steady-state peak; ≈8.8% in the stricter matched-A/B -framing). Throughput (steps/min over the window), not resolved-rate, is the -metric at this concurrency; resolved-rate deltas are within run-to-run noise. +The provider sends `x-dynamo-session-id` on every LLM request. `DYN_REQUEST_TRACE` is optional and controls Dynamo trace collection/tool-event relay only; it does not control session headers. With `DYN_REQUEST_TRACE=1` on the frontend, confirm the trace row carries the same session ID before running a larger Harbor/SWE-bench cohort. See [Agent Harnesses](/docs/agents/agent-harnesses.md#pi) for the provider contract. ## Citation @@ -181,5 +146,5 @@ ThunderAgent paper: - Conceptual docs: [docs/agents/thunderagent-router.md](/docs/agents/thunderagent-router.md) - ThunderAgent paper: - Upstream ThunderAgent reference: -- Repro fork (mini-swe-agent + session injector): +- Pi Dynamo provider: - Dynamo KV router: [Router Guide](/docs/components/router/router-guide.md) diff --git a/docs/agents/agent-harnesses.md b/docs/agents/agent-harnesses.md index cfa9ad6a6673..2dbfc05fb91c 100644 --- a/docs/agents/agent-harnesses.md +++ b/docs/agents/agent-harnesses.md @@ -35,6 +35,29 @@ codex -m zai-org/GLM-4.7-Flash -c model_provider=dynamo Codex sends a `session-id` header that Dynamo maps to `session_id`. +## Pi + +Pi uses the Dynamo provider plugin. Build and install it from the [agent-plugins](https://github.com/ai-dynamo/agent-plugins/tree/main/pi-plugin) checkout: + +```bash +git clone https://github.com/ai-dynamo/agent-plugins.git ~/agent-plugins +cd ~/agent-plugins/pi-plugin +npm install && npm run build +pi install "$PWD" +``` + +Point it at the Dynamo OpenAI-compatible endpoint and run Pi with the `dynamo` provider: + +```bash +export DYNAMO_BASE_URL=http://localhost:8000/v1 +export DYNAMO_API_KEY=dynamo-local +export DYN_AGENT_SESSION_ID=pi-demo-$(uuidgen) + +pi --model dynamo/zai-org/GLM-4.7-Flash -p 'Reply exactly ok.' +``` + +The provider sends `x-dynamo-session-id` on every LLM request and adds `x-dynamo-parent-session-id` for Pi subagents. `DYN_REQUEST_TRACE` controls optional tool-event relay and Dynamo trace collection; it does not gate the session headers. For a Harbor run, the Pi process is inside the task container, so `DYNAMO_BASE_URL` must be reachable from that container and each trial must receive a distinct `DYN_AGENT_SESSION_ID`. + ## Claude Code Claude Code uses Anthropic-compatible Messages API. The local launcher above starts `dynamo.frontend` with `--enable-anthropic-api`; for other deployments, pass that flag when starting the frontend. Then set: diff --git a/docs/agents/thunderagent-router.md b/docs/agents/thunderagent-router.md index 5b3ed562d83e..b6edfce27607 100644 --- a/docs/agents/thunderagent-router.md +++ b/docs/agents/thunderagent-router.md @@ -43,7 +43,7 @@ A program is created on its first turn, keyed by `session_id`. Public session id ## Utilization-Driven Control Loop -Pause/resume is driven by per-worker utilization — the program working set as a fraction of the worker's KV pool. The loop has three bands: +Pause/resume is driven by per-worker utilization — the program working set as a fraction of the worker's retention budget. With SGLang HiCache enabled, Dynamo reads the worker's published GPU KV and host HiCache capacities and uses their sum. The host tier is included so native GPU-to-host spill can happen before this scheduler pauses programs. Mooncake is excluded: it is conditional content-addressed storage, not guaranteed per-program retention. The loop has three bands: - At or above `pause-threshold`, the worker is over-subscribed; the tick pauses ACTING programs until utilization falls back to `pause-target`. - In the `[soft-demote-threshold, pause-threshold)` band, programs are soft-demoted (a negative priority jump) but not paused — early backpressure before a hard pause is needed. @@ -51,7 +51,7 @@ Pause/resume is driven by per-worker utilization — the program working set as | Flag | Env var | Default | Description | |---|---|---|---| -| `--pause-threshold` | `DYN_THUNDERAGENT_PAUSE_THRESHOLD` | `0.95` | Working-set fraction of the KV pool that fires a pause cycle. | +| `--pause-threshold` | `DYN_THUNDERAGENT_PAUSE_THRESHOLD` | `0.95` | Working-set fraction of the retention budget that fires a pause cycle. | | `--soft-demote-threshold` | `DYN_THUNDERAGENT_SOFT_DEMOTE_THRESHOLD` | `0.80` | Soft-demote band start (negative priority jump in `[soft, pause)`). | | `--pause-target` | `DYN_THUNDERAGENT_PAUSE_TARGET` | `0.80` | Setpoint that pause cycles drive utilization back down to. Must be `<= pause-threshold`. | | `--resume-hysteresis` | `DYN_THUNDERAGENT_RESUME_HYSTERESIS` | `0.10` | Headroom below `pause-threshold` required before any resume. | @@ -126,14 +126,14 @@ Enable these by lowering the log level for `dynamo.thunderagent_router`. They gi For per-request tracing (token counts, cache hits, worker placement), the router also integrates with [Agent Tracing](agent-tracing.md#enable-output): set `DYN_REQUEST_TRACE=1` on the frontend to land a `request_end` record per LLM call. Harness tool-event spans are separate: they require `DYN_REQUEST_TRACE_TOOL_EVENTS_ZMQ_ENDPOINT` plus a configured publisher. -## Reproducing the MiniMax-M2 Results +## Reproducing with Pi -The headline numbers (program-aware scheduling vs KV-routing-only on the same hardware, ~12-16% throughput improvement on SWE-bench-Lite with two TP4 MiniMax-M2 replicas on a single 8×H100 node) and the exact launch/repro commands live in the [folder README](../../components/src/dynamo/thunderagent_router/README.md). +The maintained smoke path is Pi through the Dynamo provider. The complete setup and session-header verification steps live in the [folder README](../../components/src/dynamo/thunderagent_router/README.md#reproducing-with-pi). Run a Harbor/SWE-bench cohort only after that smoke confirms one stable session ID reaches the router. ## References - ThunderAgent paper: [arxiv.org/abs/2602.13692](https://arxiv.org/abs/2602.13692) - Upstream ThunderAgent reference: [HaoKang-Timmy/ThunderAgent](https://github.com/HaoKang-Timmy/ThunderAgent) -- Repro fork (mini-swe-agent + session injector): [ishandhanani/ThunderAgent](https://github.com/ishandhanani/ThunderAgent) +- Pi Dynamo provider: [ai-dynamo/agent-plugins](https://github.com/ai-dynamo/agent-plugins/tree/main/pi-plugin) - Dynamo KV router: [Router Guide](../components/router/router-guide.md) - [Session IDs](session-ids.md), [Agent Tracing](agent-tracing.md), and [Agent Hints](agent-hints.md) From a9e199f8678f9cb3aa7b2b974d1ecc577a0bc08c Mon Sep 17 00:00:00 2001 From: Ishan Dhanani Date: Thu, 2 Jul 2026 20:12:43 +0000 Subject: [PATCH 03/30] docs(agents): trim Pi harness quickstart Signed-off-by: Ishan Dhanani --- docs/agents/agent-harnesses.md | 2 -- 1 file changed, 2 deletions(-) diff --git a/docs/agents/agent-harnesses.md b/docs/agents/agent-harnesses.md index 2dbfc05fb91c..a4400acb6029 100644 --- a/docs/agents/agent-harnesses.md +++ b/docs/agents/agent-harnesses.md @@ -56,8 +56,6 @@ export DYN_AGENT_SESSION_ID=pi-demo-$(uuidgen) pi --model dynamo/zai-org/GLM-4.7-Flash -p 'Reply exactly ok.' ``` -The provider sends `x-dynamo-session-id` on every LLM request and adds `x-dynamo-parent-session-id` for Pi subagents. `DYN_REQUEST_TRACE` controls optional tool-event relay and Dynamo trace collection; it does not gate the session headers. For a Harbor run, the Pi process is inside the task container, so `DYNAMO_BASE_URL` must be reachable from that container and each trial must receive a distinct `DYN_AGENT_SESSION_ID`. - ## Claude Code Claude Code uses Anthropic-compatible Messages API. The local launcher above starts `dynamo.frontend` with `--enable-anthropic-api`; for other deployments, pass that flag when starting the frontend. Then set: From fa15c3302ca9c24e0c041bf857d8d350a426392d Mon Sep 17 00:00:00 2001 From: Ishan Dhanani Date: Thu, 2 Jul 2026 20:19:24 +0000 Subject: [PATCH 04/30] docs(agents): render ThunderAgent Pi repro Signed-off-by: Ishan Dhanani --- docs/agents/thunderagent-router.md | 26 +++++++++++++++++++++++--- 1 file changed, 23 insertions(+), 3 deletions(-) diff --git a/docs/agents/thunderagent-router.md b/docs/agents/thunderagent-router.md index b6edfce27607..a171fcf9e7a8 100644 --- a/docs/agents/thunderagent-router.md +++ b/docs/agents/thunderagent-router.md @@ -5,7 +5,7 @@ title: ThunderAgent Program Scheduler subtitle: Program-level scheduling with tool-boundary pause/resume on top of KV-aware routing --- -> **Experimental — not a released component.** Run it from a source checkout, not from a `pip install ai-dynamo`. The CLI flags, session headers, and lifecycle hooks are all unstable and will change. Build and launch specifics live next to the code in [`components/src/dynamo/thunderagent_router/README.md`](../../components/src/dynamo/thunderagent_router/README.md). +> **Experimental — not a released component.** Run it from a source checkout, not from a `pip install ai-dynamo`. The CLI flags, session headers, and lifecycle hooks are all unstable and will change. `dynamo.thunderagent_router` is a standalone Dynamo router that schedules at the granularity of an agent run — the whole `LLM turn → tool call → next turn` loop — instead of individual requests. It wraps Dynamo's native KV router and adds a program-level scheduler with tool-boundary pause/resume on top of KV-aware routing, porting the scheduler from the [ThunderAgent](https://arxiv.org/abs/2602.13692) paper (Kang et al., 2026). @@ -64,7 +64,7 @@ Pause/resume is driven by per-worker utilization — the program working set as > **Constraint:** `pause-target <= pause-threshold`. The service rejects configs that violate it (along with `0 <= resume-hysteresis <= pause-threshold` and `0 <= soft-demote-threshold <= pause-threshold`). -All `KvRouter` flags from `dynamo.router` (`--router-temperature`, `--use-kv-events`, `--router-track-output-blocks`, …) are also accepted and forwarded. See the [folder README](../../components/src/dynamo/thunderagent_router/README.md) for the remaining service flags (`--endpoint`, `--model-name`, `--model-path`, tool-call and reasoning parsers). +All `KvRouter` flags from `dynamo.router` (`--router-temperature`, `--use-kv-events`, `--router-track-output-blocks`, …) are also accepted and forwarded. ## Architecture @@ -128,7 +128,27 @@ For per-request tracing (token counts, cache hits, worker placement), the router ## Reproducing with Pi -The maintained smoke path is Pi through the Dynamo provider. The complete setup and session-header verification steps live in the [folder README](../../components/src/dynamo/thunderagent_router/README.md#reproducing-with-pi). Run a Harbor/SWE-bench cohort only after that smoke confirms one stable session ID reaches the router. +The maintained smoke path is Pi through the Dynamo provider. Start a source-checkout deployment with this router and a served model, then wait for the frontend to advertise that model: + +```bash +curl -fsS http://127.0.0.1:8000/v1/models +``` + +Install the provider and send one sessionized turn through the frontend: + +```bash +git clone https://github.com/ai-dynamo/agent-plugins.git +cd agent-plugins/pi-plugin +npm install && npm run build +pi install "$PWD" + +export DYNAMO_BASE_URL=http://127.0.0.1:8000/v1 +export DYNAMO_API_KEY=dummy +export DYN_AGENT_SESSION_ID=ta-repro-$(uuidgen) +pi --model dynamo/ -p 'Reply exactly READY.' +``` + +With `DYN_REQUEST_TRACE=1` on the frontend, confirm the trace row carries the same session ID before running a larger Harbor/SWE-bench cohort. ## References From cc408722f2fc7cf1c61d41c81e0b143b000f89c4 Mon Sep 17 00:00:00 2001 From: Ishan Dhanani Date: Thu, 2 Jul 2026 20:20:15 +0000 Subject: [PATCH 05/30] docs(agents): add ThunderAgent Pi launch steps Signed-off-by: Ishan Dhanani --- docs/agents/thunderagent-router.md | 26 +++++++++++++++++++++++++- 1 file changed, 25 insertions(+), 1 deletion(-) diff --git a/docs/agents/thunderagent-router.md b/docs/agents/thunderagent-router.md index a171fcf9e7a8..de7d567515e1 100644 --- a/docs/agents/thunderagent-router.md +++ b/docs/agents/thunderagent-router.md @@ -128,7 +128,31 @@ For per-request tracing (token counts, cache hits, worker placement), the router ## Reproducing with Pi -The maintained smoke path is Pi through the Dynamo provider. Start a source-checkout deployment with this router and a served model, then wait for the frontend to advertise that model: +The maintained smoke path is one SGLang worker, this router, and Pi. In separate terminals, launch the worker, router, and frontend: + +```bash +# Terminal 1: SGLang worker with native GPU-to-host HiCache. +python -m dynamo.sglang \ + --model-path \ + --served-model-name \ + --tp \ + --enable-hierarchical-cache \ + --hicache-size \ + --hicache-write-policy write_back \ + --kv-events-config '{"publisher":"zmq","topic":"kv-events","endpoint":"tcp://*:20080","enable_kv_cache_events":true}' + +# Terminal 2: ThunderAgent program scheduler. +python -m dynamo.thunderagent_router \ + --endpoint dynamo.backend.generate \ + --model-name \ + --router-block-size 16 \ + --router-reset-states + +# Terminal 3: OpenAI-compatible frontend. +python -m dynamo.frontend --router-mode round-robin --router-reset-states +``` + +Wait for the frontend to advertise the served model: ```bash curl -fsS http://127.0.0.1:8000/v1/models From db9c368bb07a7ee31267afb867187fbbbd09e526 Mon Sep 17 00:00:00 2001 From: Ishan Dhanani Date: Thu, 2 Jul 2026 20:39:57 +0000 Subject: [PATCH 06/30] docs(agents): add Harbor Pi benchmark recipe Signed-off-by: Ishan Dhanani --- docs/agents/thunderagent-router.md | 192 ++++++++++++++++++++++++----- 1 file changed, 162 insertions(+), 30 deletions(-) diff --git a/docs/agents/thunderagent-router.md b/docs/agents/thunderagent-router.md index de7d567515e1..4dd2e9775604 100644 --- a/docs/agents/thunderagent-router.md +++ b/docs/agents/thunderagent-router.md @@ -126,53 +126,185 @@ Enable these by lowering the log level for `dynamo.thunderagent_router`. They gi For per-request tracing (token counts, cache hits, worker placement), the router also integrates with [Agent Tracing](agent-tracing.md#enable-output): set `DYN_REQUEST_TRACE=1` on the frontend to land a `request_end` record per LLM call. Harness tool-event spans are separate: they require `DYN_REQUEST_TRACE_TOOL_EVENTS_ZMQ_ENDPOINT` plus a configured publisher. -## Reproducing with Pi +## Reproducing with upstream Harbor and Pi -The maintained smoke path is one SGLang worker, this router, and Pi. In separate terminals, launch the worker, router, and frontend: +This is the maintained SWE-bench path: current upstream Harbor creates the task containers and runs Pi inside them; Dynamo, ThunderAgent, and the model stay on the host. No cache-specific engine flags are required for this smoke. + +The current upstream Harbor Pi adapter does not install external Pi providers or preserve a session while it runs Pi with `--no-session`. The [`DynamoPi` adapter](https://github.com/ai-dynamo/agent-plugins/blob/main/pi-plugin/harbor_dynamo_pi.py) adds only those two missing pieces: it installs the mounted Dynamo provider and maps Harbor's already-assigned per-trial `agent.session_id` into `DYN_AGENT_SESSION_ID`. Thus every turn of one Harbor trial carries the same `x-dynamo-session-id`; identity remains a hint, not a cache command. + +### 1. Check out the three source trees ```bash -# Terminal 1: SGLang worker with native GPU-to-host HiCache. -python -m dynamo.sglang \ - --model-path \ - --served-model-name \ - --tp \ - --enable-hierarchical-cache \ - --hicache-size \ - --hicache-write-policy write_back \ - --kv-events-config '{"publisher":"zmq","topic":"kv-events","endpoint":"tcp://*:20080","enable_kv_cache_events":true}' - -# Terminal 2: ThunderAgent program scheduler. +export DYNAMO_DIR=$HOME/src/dynamo +export HARBOR_DIR=$HOME/src/harbor +export PLUGINS_DIR=$HOME/src/agent-plugins + +git clone https://github.com/ai-dynamo/dynamo "$DYNAMO_DIR" +git clone https://github.com/harbor-framework/harbor "$HARBOR_DIR" +git clone https://github.com/ai-dynamo/agent-plugins "$PLUGINS_DIR" + +cd "$DYNAMO_DIR/lib/bindings/python" && maturin develop --uv +cd "$DYNAMO_DIR" && uv pip install -e . +cd "$HARBOR_DIR" && uv sync + +git -C "$DYNAMO_DIR" rev-parse HEAD +git -C "$HARBOR_DIR" rev-parse HEAD +git -C "$PLUGINS_DIR" rev-parse HEAD +``` + +The commands below were validated with Harbor `0.16.0`. Save the three printed revisions in the run artifact; do not substitute the old vendored ThunderAgent/Harbor launcher. + +### 2. Start the Dynamo stack + +Run this in one terminal and leave it running. File discovery, TCP request transport, and ZMQ events keep the stack self-contained: no etcd or NATS is required. + +```bash +cd "$DYNAMO_DIR" +source .venv/bin/activate + +export MODEL=zai-org/GLM-4.7-Flash +export RUN_DIR=$PWD/runs/ta-harbor-$(date -u +%Y%m%dT%H%M%SZ) +mkdir -p "$RUN_DIR/logs" "$RUN_DIR/file-kv" + +export DYN_DISCOVERY_BACKEND=file +export DYN_FILE_KV="$RUN_DIR/file-kv" +export DYN_REQUEST_PLANE=tcp +export DYN_EVENT_PLANE=zmq +export DYN_REQUEST_TRACE=1 +export DYN_REQUEST_TRACE_SINKS=jsonl_gz +export DYN_REQUEST_TRACE_OUTPUT_PATH="$RUN_DIR/request-trace" + +python -m dynamo.frontend \ + --discovery-backend file \ + --request-plane tcp \ + --event-plane zmq \ + --router-mode round-robin \ + --router-reset-states \ + --http-host 0.0.0.0 \ + --http-port 8000 \ + >"$RUN_DIR/logs/frontend.log" 2>&1 & +FRONTEND_PID=$! + +DYN_SYSTEM_PORT=8081 python -m dynamo.sglang \ + --discovery-backend file \ + --request-plane tcp \ + --event-plane zmq \ + --model-path "$MODEL" \ + --served-model-name "$MODEL" \ + --page-size 16 \ + --tp-size 2 \ + --trust-remote-code \ + --enable-metrics \ + --kv-events-config '{"publisher":"zmq","topic":"kv-events","endpoint":"tcp://*:5557","enable_kv_cache_events":true}' \ + --dyn-reasoning-parser glm45 \ + --dyn-tool-call-parser glm47 \ + >"$RUN_DIR/logs/sglang.log" 2>&1 & +SGLANG_PID=$! + python -m dynamo.thunderagent_router \ + --discovery-backend file \ + --request-plane tcp \ + --event-plane zmq \ --endpoint dynamo.backend.generate \ - --model-name \ + --model-name "$MODEL" \ + --model-path "$MODEL" \ --router-block-size 16 \ - --router-reset-states + --router-reset-states \ + --dyn-reasoning-parser glm45 \ + --dyn-tool-call-parser glm47 \ + >"$RUN_DIR/logs/thunderagent.log" 2>&1 & +THUNDERAGENT_PID=$! -# Terminal 3: OpenAI-compatible frontend. -python -m dynamo.frontend --router-mode round-robin --router-reset-states +until curl -fsS http://127.0.0.1:8000/v1/models | grep -Fq "$MODEL"; do sleep 2; done +curl -fsS http://127.0.0.1:8000/v1/models +wait "$FRONTEND_PID" "$SGLANG_PID" "$THUNDERAGENT_PID" ``` -Wait for the frontend to advertise the served model: +The final `wait` keeps the stack alive. Stop it with `Ctrl-C`; the three PIDs are available for targeted cleanup if a process exits first. + +### 3. Verify the container can reach the host frontend + +In a second terminal, choose the host address reachable from Docker containers. Do this before starting a scored task so connection failures do not become benchmark failures. ```bash -curl -fsS http://127.0.0.1:8000/v1/models +export DYNAMO_HOST=$(ip route get 1.1.1.1 | awk '{print $7; exit}') +export DYNAMO_BASE_URL="http://${DYNAMO_HOST}:8000/v1" +curl -fsS "$DYNAMO_BASE_URL/models" +docker run --rm curlimages/curl:8.11.1 -fsS "$DYNAMO_BASE_URL/models" +``` + +### 4. Run an install-only Harbor smoke + +This tests the current Harbor import path, read-only provider mount, Pi installation, and per-trial session plumbing without consuming a model rollout. + +```bash +cd "$HARBOR_DIR" +source .venv/bin/activate +export PYTHONPATH="$PLUGINS_DIR/pi-plugin${PYTHONPATH:+:$PYTHONPATH}" +export PI_PLUGIN_DIR="$PLUGINS_DIR/pi-plugin" + +harbor run \ + --dataset swebench-verified@1.0 \ + --include-task-name astropy__astropy-12907 \ + --agent harbor_dynamo_pi:DynamoPi \ + --model "dynamo/$MODEL" \ + --agent-kwarg version=0.72.1 \ + --agent-env "DYNAMO_BASE_URL=$DYNAMO_BASE_URL" \ + --agent-env DYNAMO_API_KEY=dummy \ + --agent-env DYN_REQUEST_TRACE=1 \ + --mounts "[{\"type\":\"bind\",\"source\":\"$PI_PLUGIN_DIR\",\"target\":\"/opt/pi-dynamo-provider\",\"read_only\":true}]" \ + --install-only \ + --n-concurrent 1 \ + --job-name ta-verified-install-smoke \ + --jobs-dir "$RUN_DIR/harbor" \ + --yes +``` + +### 5. Run one scored Verified task + +Remove `--install-only`; verification is enabled by default. This is the first end-to-end correctness and trace check. + +```bash +harbor run \ + --dataset swebench-verified@1.0 \ + --include-task-name astropy__astropy-12907 \ + --agent harbor_dynamo_pi:DynamoPi \ + --model "dynamo/$MODEL" \ + --agent-kwarg version=0.72.1 \ + --agent-env "DYNAMO_BASE_URL=$DYNAMO_BASE_URL" \ + --agent-env DYNAMO_API_KEY=dummy \ + --agent-env DYN_REQUEST_TRACE=1 \ + --mounts "[{\"type\":\"bind\",\"source\":\"$PI_PLUGIN_DIR\",\"target\":\"/opt/pi-dynamo-provider\",\"read_only\":true}]" \ + --n-concurrent 1 \ + --job-name ta-verified-one \ + --jobs-dir "$RUN_DIR/harbor" \ + --yes ``` -Install the provider and send one sessionized turn through the frontend: +Before scaling out, confirm both identities and scheduler behavior: + +```bash +gzip -cd "$DYN_REQUEST_TRACE_OUTPUT_PATH".*.jsonl.gz \ + | jq -r '(.event // .).agent_context.session_id // empty' \ + | sort -u + +rg 'scheduler\.tick|Paused program|Resumed program' "$RUN_DIR/logs/thunderagent.log" +find "$RUN_DIR/harbor/ta-verified-one" -name results.json -print +``` + +The trace command must report the Harbor trial session (normally `__agent`). An absence of `Paused program` or `Resumed program` is a valid low-pressure outcome, not a reason to infer scheduler activity. + +### 6. Scale to a fixed Verified cohort, then Pro + +For an exploratory load test, add `--n-tasks 32 --n-concurrent 32` to the Verified command. For a reported result, replace `--n-tasks` with repeated `--include-task-name ` arguments generated from a frozen task manifest; `--n-tasks` uses registry order and is not a stable paper cohort. + +After the same path is clean, change only the dataset selector to start a small Pro smoke: ```bash -git clone https://github.com/ai-dynamo/agent-plugins.git -cd agent-plugins/pi-plugin -npm install && npm run build -pi install "$PWD" - -export DYNAMO_BASE_URL=http://127.0.0.1:8000/v1 -export DYNAMO_API_KEY=dummy -export DYN_AGENT_SESSION_ID=ta-repro-$(uuidgen) -pi --model dynamo/ -p 'Reply exactly READY.' +--dataset swebenchpro@1.0 --n-tasks 10 --n-concurrent 10 ``` -With `DYN_REQUEST_TRACE=1` on the frontend, confirm the trace row carries the same session ID before running a larger Harbor/SWE-bench cohort. +Pre-pull task images outside the measured interval, preserve `$RUN_DIR/harbor`, `$RUN_DIR/request-trace.*.jsonl.gz`, and the three server logs, then run the policy arms against fresh server stacks. ## References From 96cd892d71754511daf06870e5365d906de776a6 Mon Sep 17 00:00:00 2001 From: Ishan Dhanani Date: Thu, 2 Jul 2026 20:41:42 +0000 Subject: [PATCH 07/30] docs(agents): clarify Harbor run environment Signed-off-by: Ishan Dhanani --- docs/agents/thunderagent-router.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/docs/agents/thunderagent-router.md b/docs/agents/thunderagent-router.md index 4dd2e9775604..2be42cbf420c 100644 --- a/docs/agents/thunderagent-router.md +++ b/docs/agents/thunderagent-router.md @@ -227,6 +227,9 @@ The final `wait` keeps the stack alive. Stop it with `Ctrl-C`; the three PIDs ar In a second terminal, choose the host address reachable from Docker containers. Do this before starting a scored task so connection failures do not become benchmark failures. ```bash +# Copy the RUN_DIR printed by the stack terminal so Harbor writes beside its traces. +export MODEL=zai-org/GLM-4.7-Flash +export RUN_DIR=/absolute/path/to/dynamo/runs/ta-harbor- export DYNAMO_HOST=$(ip route get 1.1.1.1 | awk '{print $7; exit}') export DYNAMO_BASE_URL="http://${DYNAMO_HOST}:8000/v1" curl -fsS "$DYNAMO_BASE_URL/models" From 770385ca2d3c0c3b3260da690414819367239637 Mon Sep 17 00:00:00 2001 From: Ishan Dhanani Date: Fri, 3 Jul 2026 19:24:05 +0000 Subject: [PATCH 08/30] docs(thunderagent): add Harbor Pi A/B walkthrough Signed-off-by: Ishan Dhanani --- .../src/dynamo/thunderagent_router/README.md | 314 ++++++++++++++++-- docs/agents/thunderagent-router.md | 181 +--------- 2 files changed, 296 insertions(+), 199 deletions(-) diff --git a/components/src/dynamo/thunderagent_router/README.md b/components/src/dynamo/thunderagent_router/README.md index ec472be1821a..f22ced6f5325 100644 --- a/components/src/dynamo/thunderagent_router/README.md +++ b/components/src/dynamo/thunderagent_router/README.md @@ -11,11 +11,7 @@ requests. It wraps Dynamo's native KV router and adds a program-level scheduler with tool-boundary pause/resume, porting the scheduler from the ThunderAgent paper. -**Conceptual docs live in [docs/agents/thunderagent-router.md](/docs/agents/thunderagent-router.md)** — -the scheduler model, the 5s scheduler tick (resume → pause), tool-boundary -pause/resume semantics, the utilization-driven control loop and its full knob -table, the architecture diagram, and the scheduler observability logs. This -README keeps only the build/run/repro specifics that belong next to the code. +**Conceptual docs live in [docs/agents/thunderagent-router.md](/docs/agents/thunderagent-router.md)** — the scheduler model, tool-boundary pause/resume semantics, the utilization-driven control loop, and observability. This README contains the source build and the complete Harbor/Pi A/B walkthrough. ## Install @@ -100,29 +96,309 @@ at that endpoint (producers connect) and `tool_start` / `tool_end` / `tool_call_id` pairs, giving you the full LLM-turn ↔ tool-gap timeline per agent. -## Reproducing with Pi +## Harbor/Pi A/B walkthrough -The maintained smoke path is Pi through the Dynamo provider, not the old private mini-SWE-agent fork. Start the workers, this router, and the frontend as described above, then wait for the frontend to advertise the served model: +This walkthrough runs the same SWE-bench Verified task through ThunderAgent and the stock Dynamo KV router. Harbor owns the task container, Pi runs inside it, and the model stack runs on the host. The example uses one 8-GPU node, two TP4 vLLM workers, and MiniMax-M2. There is no HiCache, Mooncake, shared cache, or frontend admission control in either arm. + +The [agent-plugins `DynamoPi` adapter](https://github.com/ai-dynamo/agent-plugins/blob/main/pi-plugin/harbor_dynamo_pi.py) is required. It installs the Dynamo provider in each Harbor task container and maps Harbor's per-trial ID to one stable `x-dynamo-session-id` across all Pi turns. The ThunderAgent arm also sends one terminal session request so the router can release the completed program; the stock KV arm disables that request because it has no lifecycle consumer. + +### 1. Check out and install Dynamo, Harbor, and the Pi adapter + +Use Python 3.12 and a machine with Docker, Node.js, and eight visible GPUs. Build Dynamo from the branch under test rather than installing a released wheel. + +```bash +export DYNAMO_DIR=$HOME/src/dynamo +export HARBOR_DIR=$HOME/src/harbor +export PLUGINS_DIR=$HOME/src/agent-plugins + +git clone https://github.com/ai-dynamo/dynamo "$DYNAMO_DIR" +git clone https://github.com/harbor-framework/harbor "$HARBOR_DIR" +git clone https://github.com/ai-dynamo/agent-plugins "$PLUGINS_DIR" + +cd "$DYNAMO_DIR" +uv venv --python 3.12 --seed .venv +source .venv/bin/activate +uv pip install pip 'maturin[patchelf]' +(cd lib/bindings/python && maturin develop --uv) +uv pip install -e lib/gpu_memory_service +uv pip install -e '.[vllm]' + +cd "$HARBOR_DIR" +uv sync + +git -C "$DYNAMO_DIR" rev-parse HEAD +git -C "$HARBOR_DIR" rev-parse HEAD +git -C "$PLUGINS_DIR" rev-parse HEAD +``` + +The Harbor commands below were validated with Harbor `0.16.0` and Pi `0.72.1`. Record all three revisions with the run artifacts. + +### 2. Start the ThunderAgent arm + +Run the following from one terminal. The public model name is registered by ThunderAgent; the two workers use an internal name so the frontend cannot bypass it. + +```bash +export DYNAMO_DIR=$HOME/src/dynamo +cd "$DYNAMO_DIR" +source .venv/bin/activate + +export MODEL_PATH=MiniMaxAI/MiniMax-M2 +export PUBLIC_MODEL=MiniMaxAI/MiniMax-M2 +export WORKER_MODEL=dyn-internal-minimax-m2 +export RUN_DIR=$DYNAMO_DIR/runs/ta-$(date -u +%Y%m%dT%H%M%SZ) +export DYN_DISCOVERY_BACKEND=file +export DYN_FILE_KV=$RUN_DIR/file-kv +export DYN_REQUEST_PLANE=tcp +export DYN_EVENT_PLANE=zmq +mkdir -p "$RUN_DIR/logs" "$DYN_FILE_KV" + +CUDA_VISIBLE_DEVICES=0,1,2,3 \ +DYN_SYSTEM_PORT=8181 \ +DYN_FORWARDPASS_METRIC_PORT=20081 \ +VLLM_NIXL_SIDE_CHANNEL_PORT=20097 \ +python -m dynamo.vllm \ + --model "$MODEL_PATH" \ + --served-model-name "$WORKER_MODEL" \ + --tensor-parallel-size 4 \ + --kv-cache-dtype fp8 \ + --block-size 16 \ + --enable-prefix-caching \ + --dyn-tool-call-parser minimax_m2 \ + --dyn-reasoning-parser minimax_append_think \ + --kv-events-config '{"publisher":"zmq","topic":"kv-events","endpoint":"tcp://*:20080","enable_kv_cache_events":true}' \ + >"$RUN_DIR/logs/worker-0.log" 2>&1 & +WORKER_0_PID=$! + +CUDA_VISIBLE_DEVICES=4,5,6,7 \ +DYN_SYSTEM_PORT=8182 \ +DYN_FORWARDPASS_METRIC_PORT=20082 \ +VLLM_NIXL_SIDE_CHANNEL_PORT=20098 \ +python -m dynamo.vllm \ + --model "$MODEL_PATH" \ + --served-model-name "$WORKER_MODEL" \ + --tensor-parallel-size 4 \ + --kv-cache-dtype fp8 \ + --block-size 16 \ + --enable-prefix-caching \ + --dyn-tool-call-parser minimax_m2 \ + --dyn-reasoning-parser minimax_append_think \ + --kv-events-config '{"publisher":"zmq","topic":"kv-events","endpoint":"tcp://*:20090","enable_kv_cache_events":true}' \ + >"$RUN_DIR/logs/worker-1.log" 2>&1 & +WORKER_1_PID=$! + +DYN_SYSTEM_PORT=8183 python -m dynamo.thunderagent_router \ + --endpoint dynamo.backend.generate \ + --model-name "$PUBLIC_MODEL" \ + --model-path "$MODEL_PATH" \ + --dyn-tool-call-parser minimax_m2 \ + --dyn-reasoning-parser minimax_append_think \ + --router-block-size 16 \ + --router-reset-states \ + --shared-cache-type none \ + --pause-threshold 0.95 \ + --pause-target 0.80 \ + --soft-demote-threshold 0.80 \ + --resume-hysteresis 0.10 \ + --resume-timeout-seconds 1800 \ + --scheduler-interval-seconds 5 \ + >"$RUN_DIR/logs/thunderagent.log" 2>&1 & +THUNDERAGENT_PID=$! + +DYN_SYSTEM_PORT=8184 python -m dynamo.frontend \ + --http-host 0.0.0.0 \ + --http-port 8100 \ + --router-mode round-robin \ + --router-reset-states \ + --shared-cache-type none \ + --admission-control none \ + >"$RUN_DIR/logs/frontend.log" 2>&1 & +FRONTEND_PID=$! + +until [ "$(curl -fsS "http://127.0.0.1:8100/v1/models/$WORKER_MODEL/ready" | jq '[.namespaces[].worker_types.aggregated.workers // 0] | add')" = 2 ]; do sleep 5; done +until curl -fsS http://127.0.0.1:8100/v1/models | grep -Fq "$PUBLIC_MODEL"; do sleep 5; done +curl -fsS "http://127.0.0.1:8100/v1/models/$WORKER_MODEL/ready" | jq +curl -fsS http://127.0.0.1:8100/v1/models | jq +echo "$RUN_DIR" +``` + +The four processes remain attached to this shell as background jobs. If readiness never succeeds, inspect the four files in `$RUN_DIR/logs` instead of restarting blindly. + +### 3. Run the ThunderAgent task through Harbor + +Run this from a second terminal. `DYNAMO_BASE_URL` must use an address reachable from the task container, not `127.0.0.1` under Docker's default bridge network. + +```bash +export HARBOR_DIR=$HOME/src/harbor +export PLUGINS_DIR=$HOME/src/agent-plugins +cd "$HARBOR_DIR" +source .venv/bin/activate + +export PUBLIC_MODEL=MiniMaxAI/MiniMax-M2 +export RUN_DIR=/absolute/path/printed/by/the/stack/terminal +export PI_PLUGIN_DIR=$PLUGINS_DIR/pi-plugin +export PYTHONPATH=$PI_PLUGIN_DIR${PYTHONPATH:+:$PYTHONPATH} +export DYNAMO_HOST=$(ip route get 1.1.1.1 | awk '{print $7; exit}') +export DYNAMO_BASE_URL=http://$DYNAMO_HOST:8100/v1 + +curl -fsS "$DYNAMO_BASE_URL/models" | jq +docker run --rm curlimages/curl:8.11.1 -fsS "$DYNAMO_BASE_URL/models" + +harbor run \ + --dataset swebench-verified@1.0 \ + --include-task-name astropy__astropy-12907 \ + --agent harbor_dynamo_pi:DynamoPi \ + --model "dynamo/$PUBLIC_MODEL" \ + --agent-kwarg version=0.72.1 \ + --agent-env "DYNAMO_BASE_URL=$DYNAMO_BASE_URL" \ + --agent-env DYNAMO_API_KEY=dynamo-local \ + --agent-env DYN_AGENT_SESSION_FINAL=1 \ + --mounts "[{\"type\":\"bind\",\"source\":\"$PI_PLUGIN_DIR\",\"target\":\"/opt/pi-dynamo-provider\",\"read_only\":true}]" \ + --n-concurrent 1 \ + --n-concurrent-agents 1 \ + --max-retries 0 \ + --agent-setup-timeout-multiplier 10 \ + --no-force-build \ + --no-delete \ + --environment-kwarg keep_containers=true \ + --jobs-dir "$RUN_DIR/harbor" \ + --job-name ta-verified-one \ + --yes +``` + +`--no-delete --environment-kwarg keep_containers=true` stops completed task containers without running `docker compose down` on the hot path. This avoids completed containers holding Harbor concurrency slots under load and preserves them for inspection. + +Check that the task produced Pi output and that ThunderAgent observed the lifecycle: + +```bash +find "$RUN_DIR/harbor/ta-verified-one" -name pi.txt -o -name results.json +rg 'scheduler\.tick|Paused program|Resumed program|terminal' "$RUN_DIR/logs/thunderagent.log" +``` + +No pause/resume lines are expected from a one-task smoke; those only appear after the working set reaches the configured thresholds. + +### 4. Stop ThunderAgent and start the stock KV arm + +Return to the first terminal and stop every process from the first arm. A fair comparison starts fresh workers and an empty file-discovery directory. + +```bash +kill "$FRONTEND_PID" "$THUNDERAGENT_PID" "$WORKER_0_PID" "$WORKER_1_PID" +wait "$FRONTEND_PID" "$THUNDERAGENT_PID" "$WORKER_0_PID" "$WORKER_1_PID" 2>/dev/null || true +docker ps -a --filter status=exited +# On a dedicated benchmark host, remove the stopped containers from the TA job. +docker container prune -f + +export WORKER_MODEL=MiniMaxAI/MiniMax-M2 +export RUN_DIR=$DYNAMO_DIR/runs/kv-$(date -u +%Y%m%dT%H%M%SZ) +export DYN_FILE_KV=$RUN_DIR/file-kv +mkdir -p "$RUN_DIR/logs" "$DYN_FILE_KV" + +CUDA_VISIBLE_DEVICES=0,1,2,3 \ +DYN_SYSTEM_PORT=8181 \ +DYN_FORWARDPASS_METRIC_PORT=20081 \ +VLLM_NIXL_SIDE_CHANNEL_PORT=20097 \ +python -m dynamo.vllm \ + --model "$MODEL_PATH" \ + --served-model-name "$WORKER_MODEL" \ + --tensor-parallel-size 4 \ + --kv-cache-dtype fp8 \ + --block-size 16 \ + --enable-prefix-caching \ + --dyn-tool-call-parser minimax_m2 \ + --dyn-reasoning-parser minimax_append_think \ + --kv-events-config '{"publisher":"zmq","topic":"kv-events","endpoint":"tcp://*:20080","enable_kv_cache_events":true}' \ + >"$RUN_DIR/logs/worker-0.log" 2>&1 & +WORKER_0_PID=$! + +CUDA_VISIBLE_DEVICES=4,5,6,7 \ +DYN_SYSTEM_PORT=8182 \ +DYN_FORWARDPASS_METRIC_PORT=20082 \ +VLLM_NIXL_SIDE_CHANNEL_PORT=20098 \ +python -m dynamo.vllm \ + --model "$MODEL_PATH" \ + --served-model-name "$WORKER_MODEL" \ + --tensor-parallel-size 4 \ + --kv-cache-dtype fp8 \ + --block-size 16 \ + --enable-prefix-caching \ + --dyn-tool-call-parser minimax_m2 \ + --dyn-reasoning-parser minimax_append_think \ + --kv-events-config '{"publisher":"zmq","topic":"kv-events","endpoint":"tcp://*:20090","enable_kv_cache_events":true}' \ + >"$RUN_DIR/logs/worker-1.log" 2>&1 & +WORKER_1_PID=$! + +DYN_SYSTEM_PORT=8184 python -m dynamo.frontend \ + --http-host 0.0.0.0 \ + --http-port 8100 \ + --router-mode kv \ + --router-reset-states \ + --shared-cache-type none \ + --admission-control none \ + >"$RUN_DIR/logs/frontend.log" 2>&1 & +FRONTEND_PID=$! + +until [ "$(curl -fsS "http://127.0.0.1:8100/v1/models/$PUBLIC_MODEL/ready" | jq '[.namespaces[].worker_types.aggregated.workers // 0] | add')" = 2 ]; do sleep 5; done +curl -fsS "http://127.0.0.1:8100/v1/models/$PUBLIC_MODEL/ready" | jq +curl -fsS http://127.0.0.1:8100/v1/models | jq +echo "$RUN_DIR" +``` + +### 5. Run the same task through stock KV routing + +In the Harbor terminal, point `RUN_DIR` at the stock arm and run the same command with terminal control disabled: + +```bash +export RUN_DIR=/absolute/path/printed/by/the/stock/stack/terminal + +harbor run \ + --dataset swebench-verified@1.0 \ + --include-task-name astropy__astropy-12907 \ + --agent harbor_dynamo_pi:DynamoPi \ + --model "dynamo/$PUBLIC_MODEL" \ + --agent-kwarg version=0.72.1 \ + --agent-env "DYNAMO_BASE_URL=$DYNAMO_BASE_URL" \ + --agent-env DYNAMO_API_KEY=dynamo-local \ + --agent-env DYN_AGENT_SESSION_FINAL=0 \ + --mounts "[{\"type\":\"bind\",\"source\":\"$PI_PLUGIN_DIR\",\"target\":\"/opt/pi-dynamo-provider\",\"read_only\":true}]" \ + --n-concurrent 1 \ + --n-concurrent-agents 1 \ + --max-retries 0 \ + --agent-setup-timeout-multiplier 10 \ + --no-force-build \ + --no-delete \ + --environment-kwarg keep_containers=true \ + --jobs-dir "$RUN_DIR/harbor" \ + --job-name kv-verified-one \ + --yes + +find "$RUN_DIR/harbor/kv-verified-one" -name pi.txt -o -name results.json +``` + +The two Harbor commands differ only in the job name and `DYN_AGENT_SESSION_FINAL`. Stable session headers are still sent in the stock arm; they are identity metadata and do not enable ThunderAgent or sticky routing. + +### 6. Scale the comparison + +After both one-task smokes pass, use the same frozen task list and concurrency for both arms. For an exploratory 32-task run, replace `--include-task-name ...` with `--n-tasks 32` and set both concurrency options to 32. For a reported result, use repeated `--include-task-name ` arguments from a checked-in manifest because registry order is not a stable cohort. Add `--disable-verification` only when measuring serving throughput rather than solve rate. + +Start Tachometer before each Harbor run if you need server-side metrics: ```bash -curl -fsS http://127.0.0.1:8000/v1/models +tachometer-scraper \ + --endpoint frontend=http://127.0.0.1:8100/metrics \ + --endpoint worker0=http://127.0.0.1:8181/metrics \ + --endpoint worker1=http://127.0.0.1:8182/metrics \ + --freq 1.0 \ + --storage "$RUN_DIR/tachometer" ``` -Install the [Pi Dynamo provider](https://github.com/ai-dynamo/agent-plugins/tree/main/pi-plugin) from a checkout, then send a sessionized turn through the frontend: +Add `--endpoint thunderagent=http://127.0.0.1:8183/metrics` for the ThunderAgent arm. Run each arm from fresh server processes, pre-pull task images before the measured interval, and preserve the Harbor job directory and server logs. Stop the stock processes when finished: ```bash -git clone https://github.com/ai-dynamo/agent-plugins.git -cd agent-plugins/pi-plugin -npm install && npm run build -pi install "$PWD" - -export DYNAMO_BASE_URL=http://127.0.0.1:8000/v1 -export DYNAMO_API_KEY=dummy -export DYN_AGENT_SESSION_ID=ta-repro-$(uuidgen) -pi --model dynamo/ -p 'Reply exactly READY.' +kill "$FRONTEND_PID" "$WORKER_0_PID" "$WORKER_1_PID" +wait "$FRONTEND_PID" "$WORKER_0_PID" "$WORKER_1_PID" 2>/dev/null || true +docker ps -a --filter status=exited ``` -The provider sends `x-dynamo-session-id` on every LLM request. `DYN_REQUEST_TRACE` is optional and controls Dynamo trace collection/tool-event relay only; it does not control session headers. With `DYN_REQUEST_TRACE=1` on the frontend, confirm the trace row carries the same session ID before running a larger Harbor/SWE-bench cohort. See [Agent Harnesses](/docs/agents/agent-harnesses.md#pi) for the provider contract. +Inspect the stopped Harbor containers before removing the ones belonging to the completed jobs. ## Citation diff --git a/docs/agents/thunderagent-router.md b/docs/agents/thunderagent-router.md index 2be42cbf420c..6a54a7c77590 100644 --- a/docs/agents/thunderagent-router.md +++ b/docs/agents/thunderagent-router.md @@ -128,186 +128,7 @@ For per-request tracing (token counts, cache hits, worker placement), the router ## Reproducing with upstream Harbor and Pi -This is the maintained SWE-bench path: current upstream Harbor creates the task containers and runs Pi inside them; Dynamo, ThunderAgent, and the model stay on the host. No cache-specific engine flags are required for this smoke. - -The current upstream Harbor Pi adapter does not install external Pi providers or preserve a session while it runs Pi with `--no-session`. The [`DynamoPi` adapter](https://github.com/ai-dynamo/agent-plugins/blob/main/pi-plugin/harbor_dynamo_pi.py) adds only those two missing pieces: it installs the mounted Dynamo provider and maps Harbor's already-assigned per-trial `agent.session_id` into `DYN_AGENT_SESSION_ID`. Thus every turn of one Harbor trial carries the same `x-dynamo-session-id`; identity remains a hint, not a cache command. - -### 1. Check out the three source trees - -```bash -export DYNAMO_DIR=$HOME/src/dynamo -export HARBOR_DIR=$HOME/src/harbor -export PLUGINS_DIR=$HOME/src/agent-plugins - -git clone https://github.com/ai-dynamo/dynamo "$DYNAMO_DIR" -git clone https://github.com/harbor-framework/harbor "$HARBOR_DIR" -git clone https://github.com/ai-dynamo/agent-plugins "$PLUGINS_DIR" - -cd "$DYNAMO_DIR/lib/bindings/python" && maturin develop --uv -cd "$DYNAMO_DIR" && uv pip install -e . -cd "$HARBOR_DIR" && uv sync - -git -C "$DYNAMO_DIR" rev-parse HEAD -git -C "$HARBOR_DIR" rev-parse HEAD -git -C "$PLUGINS_DIR" rev-parse HEAD -``` - -The commands below were validated with Harbor `0.16.0`. Save the three printed revisions in the run artifact; do not substitute the old vendored ThunderAgent/Harbor launcher. - -### 2. Start the Dynamo stack - -Run this in one terminal and leave it running. File discovery, TCP request transport, and ZMQ events keep the stack self-contained: no etcd or NATS is required. - -```bash -cd "$DYNAMO_DIR" -source .venv/bin/activate - -export MODEL=zai-org/GLM-4.7-Flash -export RUN_DIR=$PWD/runs/ta-harbor-$(date -u +%Y%m%dT%H%M%SZ) -mkdir -p "$RUN_DIR/logs" "$RUN_DIR/file-kv" - -export DYN_DISCOVERY_BACKEND=file -export DYN_FILE_KV="$RUN_DIR/file-kv" -export DYN_REQUEST_PLANE=tcp -export DYN_EVENT_PLANE=zmq -export DYN_REQUEST_TRACE=1 -export DYN_REQUEST_TRACE_SINKS=jsonl_gz -export DYN_REQUEST_TRACE_OUTPUT_PATH="$RUN_DIR/request-trace" - -python -m dynamo.frontend \ - --discovery-backend file \ - --request-plane tcp \ - --event-plane zmq \ - --router-mode round-robin \ - --router-reset-states \ - --http-host 0.0.0.0 \ - --http-port 8000 \ - >"$RUN_DIR/logs/frontend.log" 2>&1 & -FRONTEND_PID=$! - -DYN_SYSTEM_PORT=8081 python -m dynamo.sglang \ - --discovery-backend file \ - --request-plane tcp \ - --event-plane zmq \ - --model-path "$MODEL" \ - --served-model-name "$MODEL" \ - --page-size 16 \ - --tp-size 2 \ - --trust-remote-code \ - --enable-metrics \ - --kv-events-config '{"publisher":"zmq","topic":"kv-events","endpoint":"tcp://*:5557","enable_kv_cache_events":true}' \ - --dyn-reasoning-parser glm45 \ - --dyn-tool-call-parser glm47 \ - >"$RUN_DIR/logs/sglang.log" 2>&1 & -SGLANG_PID=$! - -python -m dynamo.thunderagent_router \ - --discovery-backend file \ - --request-plane tcp \ - --event-plane zmq \ - --endpoint dynamo.backend.generate \ - --model-name "$MODEL" \ - --model-path "$MODEL" \ - --router-block-size 16 \ - --router-reset-states \ - --dyn-reasoning-parser glm45 \ - --dyn-tool-call-parser glm47 \ - >"$RUN_DIR/logs/thunderagent.log" 2>&1 & -THUNDERAGENT_PID=$! - -until curl -fsS http://127.0.0.1:8000/v1/models | grep -Fq "$MODEL"; do sleep 2; done -curl -fsS http://127.0.0.1:8000/v1/models -wait "$FRONTEND_PID" "$SGLANG_PID" "$THUNDERAGENT_PID" -``` - -The final `wait` keeps the stack alive. Stop it with `Ctrl-C`; the three PIDs are available for targeted cleanup if a process exits first. - -### 3. Verify the container can reach the host frontend - -In a second terminal, choose the host address reachable from Docker containers. Do this before starting a scored task so connection failures do not become benchmark failures. - -```bash -# Copy the RUN_DIR printed by the stack terminal so Harbor writes beside its traces. -export MODEL=zai-org/GLM-4.7-Flash -export RUN_DIR=/absolute/path/to/dynamo/runs/ta-harbor- -export DYNAMO_HOST=$(ip route get 1.1.1.1 | awk '{print $7; exit}') -export DYNAMO_BASE_URL="http://${DYNAMO_HOST}:8000/v1" -curl -fsS "$DYNAMO_BASE_URL/models" -docker run --rm curlimages/curl:8.11.1 -fsS "$DYNAMO_BASE_URL/models" -``` - -### 4. Run an install-only Harbor smoke - -This tests the current Harbor import path, read-only provider mount, Pi installation, and per-trial session plumbing without consuming a model rollout. - -```bash -cd "$HARBOR_DIR" -source .venv/bin/activate -export PYTHONPATH="$PLUGINS_DIR/pi-plugin${PYTHONPATH:+:$PYTHONPATH}" -export PI_PLUGIN_DIR="$PLUGINS_DIR/pi-plugin" - -harbor run \ - --dataset swebench-verified@1.0 \ - --include-task-name astropy__astropy-12907 \ - --agent harbor_dynamo_pi:DynamoPi \ - --model "dynamo/$MODEL" \ - --agent-kwarg version=0.72.1 \ - --agent-env "DYNAMO_BASE_URL=$DYNAMO_BASE_URL" \ - --agent-env DYNAMO_API_KEY=dummy \ - --agent-env DYN_REQUEST_TRACE=1 \ - --mounts "[{\"type\":\"bind\",\"source\":\"$PI_PLUGIN_DIR\",\"target\":\"/opt/pi-dynamo-provider\",\"read_only\":true}]" \ - --install-only \ - --n-concurrent 1 \ - --job-name ta-verified-install-smoke \ - --jobs-dir "$RUN_DIR/harbor" \ - --yes -``` - -### 5. Run one scored Verified task - -Remove `--install-only`; verification is enabled by default. This is the first end-to-end correctness and trace check. - -```bash -harbor run \ - --dataset swebench-verified@1.0 \ - --include-task-name astropy__astropy-12907 \ - --agent harbor_dynamo_pi:DynamoPi \ - --model "dynamo/$MODEL" \ - --agent-kwarg version=0.72.1 \ - --agent-env "DYNAMO_BASE_URL=$DYNAMO_BASE_URL" \ - --agent-env DYNAMO_API_KEY=dummy \ - --agent-env DYN_REQUEST_TRACE=1 \ - --mounts "[{\"type\":\"bind\",\"source\":\"$PI_PLUGIN_DIR\",\"target\":\"/opt/pi-dynamo-provider\",\"read_only\":true}]" \ - --n-concurrent 1 \ - --job-name ta-verified-one \ - --jobs-dir "$RUN_DIR/harbor" \ - --yes -``` - -Before scaling out, confirm both identities and scheduler behavior: - -```bash -gzip -cd "$DYN_REQUEST_TRACE_OUTPUT_PATH".*.jsonl.gz \ - | jq -r '(.event // .).agent_context.session_id // empty' \ - | sort -u - -rg 'scheduler\.tick|Paused program|Resumed program' "$RUN_DIR/logs/thunderagent.log" -find "$RUN_DIR/harbor/ta-verified-one" -name results.json -print -``` - -The trace command must report the Harbor trial session (normally `__agent`). An absence of `Paused program` or `Resumed program` is a valid low-pressure outcome, not a reason to infer scheduler activity. - -### 6. Scale to a fixed Verified cohort, then Pro - -For an exploratory load test, add `--n-tasks 32 --n-concurrent 32` to the Verified command. For a reported result, replace `--n-tasks` with repeated `--include-task-name ` arguments generated from a frozen task manifest; `--n-tasks` uses registry order and is not a stable paper cohort. - -After the same path is clean, change only the dataset selector to start a small Pro smoke: - -```bash ---dataset swebenchpro@1.0 --n-tasks 10 --n-concurrent 10 -``` - -Pre-pull task images outside the measured interval, preserve `$RUN_DIR/harbor`, `$RUN_DIR/request-trace.*.jsonl.gz`, and the three server logs, then run the policy arms against fresh server stacks. +The maintained end-to-end path uses upstream Harbor to create SWE-bench containers and Pi with the Dynamo provider inside each container. The complete source build, ThunderAgent arm, stock KV arm, stable-session setup, and scaling procedure live next to the experimental component in [`components/src/dynamo/thunderagent_router/README.md`](/components/src/dynamo/thunderagent_router/README.md#harborpi-ab-walkthrough). ## References From 98326978a4781322cd4258ef7aaa11c3ded0c64c Mon Sep 17 00:00:00 2001 From: Ishan Dhanani Date: Sat, 4 Jul 2026 00:07:22 +0000 Subject: [PATCH 09/30] docs(thunderagent): simplify Harbor Pi walkthrough Signed-off-by: Ishan Dhanani --- .../src/dynamo/thunderagent_router/README.md | 276 +++--------------- .../thunderagent_router/run_minimax_8xh100.sh | 95 ++++-- 2 files changed, 103 insertions(+), 268 deletions(-) diff --git a/components/src/dynamo/thunderagent_router/README.md b/components/src/dynamo/thunderagent_router/README.md index f22ced6f5325..1ff749a46895 100644 --- a/components/src/dynamo/thunderagent_router/README.md +++ b/components/src/dynamo/thunderagent_router/README.md @@ -102,20 +102,16 @@ This walkthrough runs the same SWE-bench Verified task through ThunderAgent and The [agent-plugins `DynamoPi` adapter](https://github.com/ai-dynamo/agent-plugins/blob/main/pi-plugin/harbor_dynamo_pi.py) is required. It installs the Dynamo provider in each Harbor task container and maps Harbor's per-trial ID to one stable `x-dynamo-session-id` across all Pi turns. The ThunderAgent arm also sends one terminal session request so the router can release the completed program; the stock KV arm disables that request because it has no lifecycle consumer. -### 1. Check out and install Dynamo, Harbor, and the Pi adapter +### 1. Install the three source trees Use Python 3.12 and a machine with Docker, Node.js, and eight visible GPUs. Build Dynamo from the branch under test rather than installing a released wheel. ```bash -export DYNAMO_DIR=$HOME/src/dynamo -export HARBOR_DIR=$HOME/src/harbor -export PLUGINS_DIR=$HOME/src/agent-plugins +git clone https://github.com/ai-dynamo/dynamo ~/src/dynamo +git clone https://github.com/harbor-framework/harbor ~/src/harbor +git clone https://github.com/ai-dynamo/agent-plugins ~/src/agent-plugins -git clone https://github.com/ai-dynamo/dynamo "$DYNAMO_DIR" -git clone https://github.com/harbor-framework/harbor "$HARBOR_DIR" -git clone https://github.com/ai-dynamo/agent-plugins "$PLUGINS_DIR" - -cd "$DYNAMO_DIR" +cd ~/src/dynamo uv venv --python 3.12 --seed .venv source .venv/bin/activate uv pip install pip 'maturin[patchelf]' @@ -123,263 +119,69 @@ uv pip install pip 'maturin[patchelf]' uv pip install -e lib/gpu_memory_service uv pip install -e '.[vllm]' -cd "$HARBOR_DIR" +cd ~/src/harbor uv sync - -git -C "$DYNAMO_DIR" rev-parse HEAD -git -C "$HARBOR_DIR" rev-parse HEAD -git -C "$PLUGINS_DIR" rev-parse HEAD ``` -The Harbor commands below were validated with Harbor `0.16.0` and Pi `0.72.1`. Record all three revisions with the run artifacts. +These commands were validated with Harbor `0.16.0` and Pi `0.72.1`. Record the Dynamo, Harbor, and agent-plugins revisions with benchmark artifacts. -### 2. Start the ThunderAgent arm +### 2. Run the ThunderAgent smoke -Run the following from one terminal. The public model name is registered by ThunderAgent; the two workers use an internal name so the frontend cannot bypass it. +The checked-in launcher starts both TP4 workers, ThunderAgent, and the frontend. It prints `ta stack ready` only after both workers register. ```bash -export DYNAMO_DIR=$HOME/src/dynamo -cd "$DYNAMO_DIR" +cd ~/src/dynamo source .venv/bin/activate - -export MODEL_PATH=MiniMaxAI/MiniMax-M2 -export PUBLIC_MODEL=MiniMaxAI/MiniMax-M2 -export WORKER_MODEL=dyn-internal-minimax-m2 -export RUN_DIR=$DYNAMO_DIR/runs/ta-$(date -u +%Y%m%dT%H%M%SZ) -export DYN_DISCOVERY_BACKEND=file -export DYN_FILE_KV=$RUN_DIR/file-kv -export DYN_REQUEST_PLANE=tcp -export DYN_EVENT_PLANE=zmq -mkdir -p "$RUN_DIR/logs" "$DYN_FILE_KV" - -CUDA_VISIBLE_DEVICES=0,1,2,3 \ -DYN_SYSTEM_PORT=8181 \ -DYN_FORWARDPASS_METRIC_PORT=20081 \ -VLLM_NIXL_SIDE_CHANNEL_PORT=20097 \ -python -m dynamo.vllm \ - --model "$MODEL_PATH" \ - --served-model-name "$WORKER_MODEL" \ - --tensor-parallel-size 4 \ - --kv-cache-dtype fp8 \ - --block-size 16 \ - --enable-prefix-caching \ - --dyn-tool-call-parser minimax_m2 \ - --dyn-reasoning-parser minimax_append_think \ - --kv-events-config '{"publisher":"zmq","topic":"kv-events","endpoint":"tcp://*:20080","enable_kv_cache_events":true}' \ - >"$RUN_DIR/logs/worker-0.log" 2>&1 & -WORKER_0_PID=$! - -CUDA_VISIBLE_DEVICES=4,5,6,7 \ -DYN_SYSTEM_PORT=8182 \ -DYN_FORWARDPASS_METRIC_PORT=20082 \ -VLLM_NIXL_SIDE_CHANNEL_PORT=20098 \ -python -m dynamo.vllm \ - --model "$MODEL_PATH" \ - --served-model-name "$WORKER_MODEL" \ - --tensor-parallel-size 4 \ - --kv-cache-dtype fp8 \ - --block-size 16 \ - --enable-prefix-caching \ - --dyn-tool-call-parser minimax_m2 \ - --dyn-reasoning-parser minimax_append_think \ - --kv-events-config '{"publisher":"zmq","topic":"kv-events","endpoint":"tcp://*:20090","enable_kv_cache_events":true}' \ - >"$RUN_DIR/logs/worker-1.log" 2>&1 & -WORKER_1_PID=$! - -DYN_SYSTEM_PORT=8183 python -m dynamo.thunderagent_router \ - --endpoint dynamo.backend.generate \ - --model-name "$PUBLIC_MODEL" \ - --model-path "$MODEL_PATH" \ - --dyn-tool-call-parser minimax_m2 \ - --dyn-reasoning-parser minimax_append_think \ - --router-block-size 16 \ - --router-reset-states \ - --shared-cache-type none \ - --pause-threshold 0.95 \ - --pause-target 0.80 \ - --soft-demote-threshold 0.80 \ - --resume-hysteresis 0.10 \ - --resume-timeout-seconds 1800 \ - --scheduler-interval-seconds 5 \ - >"$RUN_DIR/logs/thunderagent.log" 2>&1 & -THUNDERAGENT_PID=$! - -DYN_SYSTEM_PORT=8184 python -m dynamo.frontend \ - --http-host 0.0.0.0 \ - --http-port 8100 \ - --router-mode round-robin \ - --router-reset-states \ - --shared-cache-type none \ - --admission-control none \ - >"$RUN_DIR/logs/frontend.log" 2>&1 & -FRONTEND_PID=$! - -until [ "$(curl -fsS "http://127.0.0.1:8100/v1/models/$WORKER_MODEL/ready" | jq '[.namespaces[].worker_types.aggregated.workers // 0] | add')" = 2 ]; do sleep 5; done -until curl -fsS http://127.0.0.1:8100/v1/models | grep -Fq "$PUBLIC_MODEL"; do sleep 5; done -curl -fsS "http://127.0.0.1:8100/v1/models/$WORKER_MODEL/ready" | jq -curl -fsS http://127.0.0.1:8100/v1/models | jq -echo "$RUN_DIR" +./components/src/dynamo/thunderagent_router/run_minimax_8xh100.sh ta ``` -The four processes remain attached to this shell as background jobs. If readiness never succeeds, inspect the four files in `$RUN_DIR/logs` instead of restarting blindly. - -### 3. Run the ThunderAgent task through Harbor - -Run this from a second terminal. `DYNAMO_BASE_URL` must use an address reachable from the task container, not `127.0.0.1` under Docker's default bridge network. +In a second terminal, run one Verified task. Use a host address reachable from Docker rather than `127.0.0.1`. ```bash -export HARBOR_DIR=$HOME/src/harbor -export PLUGINS_DIR=$HOME/src/agent-plugins -cd "$HARBOR_DIR" +cd ~/src/harbor source .venv/bin/activate -export PUBLIC_MODEL=MiniMaxAI/MiniMax-M2 -export RUN_DIR=/absolute/path/printed/by/the/stack/terminal -export PI_PLUGIN_DIR=$PLUGINS_DIR/pi-plugin -export PYTHONPATH=$PI_PLUGIN_DIR${PYTHONPATH:+:$PYTHONPATH} -export DYNAMO_HOST=$(ip route get 1.1.1.1 | awk '{print $7; exit}') -export DYNAMO_BASE_URL=http://$DYNAMO_HOST:8100/v1 - -curl -fsS "$DYNAMO_BASE_URL/models" | jq +export PI_PLUGIN_DIR=~/src/agent-plugins/pi-plugin +export PYTHONPATH=$PI_PLUGIN_DIR +export DYNAMO_BASE_URL=http://$(ip route get 1.1.1.1 | awk '{print $7; exit}'):8100/v1 docker run --rm curlimages/curl:8.11.1 -fsS "$DYNAMO_BASE_URL/models" harbor run \ - --dataset swebench-verified@1.0 \ - --include-task-name astropy__astropy-12907 \ - --agent harbor_dynamo_pi:DynamoPi \ - --model "dynamo/$PUBLIC_MODEL" \ - --agent-kwarg version=0.72.1 \ - --agent-env "DYNAMO_BASE_URL=$DYNAMO_BASE_URL" \ - --agent-env DYNAMO_API_KEY=dynamo-local \ - --agent-env DYN_AGENT_SESSION_FINAL=1 \ + -d swebench-verified@1.0 -i astropy__astropy-12907 \ + -a harbor_dynamo_pi:DynamoPi -m dynamo/MiniMaxAI/MiniMax-M2 \ + --ak version=0.72.1 --ae "DYNAMO_BASE_URL=$DYNAMO_BASE_URL" \ --mounts "[{\"type\":\"bind\",\"source\":\"$PI_PLUGIN_DIR\",\"target\":\"/opt/pi-dynamo-provider\",\"read_only\":true}]" \ - --n-concurrent 1 \ - --n-concurrent-agents 1 \ - --max-retries 0 \ - --agent-setup-timeout-multiplier 10 \ - --no-force-build \ - --no-delete \ - --environment-kwarg keep_containers=true \ - --jobs-dir "$RUN_DIR/harbor" \ - --job-name ta-verified-one \ - --yes -``` - -`--no-delete --environment-kwarg keep_containers=true` stops completed task containers without running `docker compose down` on the hot path. This avoids completed containers holding Harbor concurrency slots under load and preserves them for inspection. - -Check that the task produced Pi output and that ThunderAgent observed the lifecycle: - -```bash -find "$RUN_DIR/harbor/ta-verified-one" -name pi.txt -o -name results.json -rg 'scheduler\.tick|Paused program|Resumed program|terminal' "$RUN_DIR/logs/thunderagent.log" + -n 1 --agent-setup-timeout-multiplier 10 \ + --job-name ta-verified-one -y ``` No pause/resume lines are expected from a one-task smoke; those only appear after the working set reaches the configured thresholds. -### 4. Stop ThunderAgent and start the stock KV arm +### 3. Run the stock KV smoke -Return to the first terminal and stop every process from the first arm. A fair comparison starts fresh workers and an empty file-discovery directory. +Stop the ThunderAgent launcher with `Ctrl-C`, then start a fresh stock-KV stack: ```bash -kill "$FRONTEND_PID" "$THUNDERAGENT_PID" "$WORKER_0_PID" "$WORKER_1_PID" -wait "$FRONTEND_PID" "$THUNDERAGENT_PID" "$WORKER_0_PID" "$WORKER_1_PID" 2>/dev/null || true -docker ps -a --filter status=exited -# On a dedicated benchmark host, remove the stopped containers from the TA job. -docker container prune -f - -export WORKER_MODEL=MiniMaxAI/MiniMax-M2 -export RUN_DIR=$DYNAMO_DIR/runs/kv-$(date -u +%Y%m%dT%H%M%SZ) -export DYN_FILE_KV=$RUN_DIR/file-kv -mkdir -p "$RUN_DIR/logs" "$DYN_FILE_KV" - -CUDA_VISIBLE_DEVICES=0,1,2,3 \ -DYN_SYSTEM_PORT=8181 \ -DYN_FORWARDPASS_METRIC_PORT=20081 \ -VLLM_NIXL_SIDE_CHANNEL_PORT=20097 \ -python -m dynamo.vllm \ - --model "$MODEL_PATH" \ - --served-model-name "$WORKER_MODEL" \ - --tensor-parallel-size 4 \ - --kv-cache-dtype fp8 \ - --block-size 16 \ - --enable-prefix-caching \ - --dyn-tool-call-parser minimax_m2 \ - --dyn-reasoning-parser minimax_append_think \ - --kv-events-config '{"publisher":"zmq","topic":"kv-events","endpoint":"tcp://*:20080","enable_kv_cache_events":true}' \ - >"$RUN_DIR/logs/worker-0.log" 2>&1 & -WORKER_0_PID=$! - -CUDA_VISIBLE_DEVICES=4,5,6,7 \ -DYN_SYSTEM_PORT=8182 \ -DYN_FORWARDPASS_METRIC_PORT=20082 \ -VLLM_NIXL_SIDE_CHANNEL_PORT=20098 \ -python -m dynamo.vllm \ - --model "$MODEL_PATH" \ - --served-model-name "$WORKER_MODEL" \ - --tensor-parallel-size 4 \ - --kv-cache-dtype fp8 \ - --block-size 16 \ - --enable-prefix-caching \ - --dyn-tool-call-parser minimax_m2 \ - --dyn-reasoning-parser minimax_append_think \ - --kv-events-config '{"publisher":"zmq","topic":"kv-events","endpoint":"tcp://*:20090","enable_kv_cache_events":true}' \ - >"$RUN_DIR/logs/worker-1.log" 2>&1 & -WORKER_1_PID=$! - -DYN_SYSTEM_PORT=8184 python -m dynamo.frontend \ - --http-host 0.0.0.0 \ - --http-port 8100 \ - --router-mode kv \ - --router-reset-states \ - --shared-cache-type none \ - --admission-control none \ - >"$RUN_DIR/logs/frontend.log" 2>&1 & -FRONTEND_PID=$! - -until [ "$(curl -fsS "http://127.0.0.1:8100/v1/models/$PUBLIC_MODEL/ready" | jq '[.namespaces[].worker_types.aggregated.workers // 0] | add')" = 2 ]; do sleep 5; done -curl -fsS "http://127.0.0.1:8100/v1/models/$PUBLIC_MODEL/ready" | jq -curl -fsS http://127.0.0.1:8100/v1/models | jq -echo "$RUN_DIR" +cd ~/src/dynamo +source .venv/bin/activate +./components/src/dynamo/thunderagent_router/run_minimax_8xh100.sh kv ``` -### 5. Run the same task through stock KV routing - -In the Harbor terminal, point `RUN_DIR` at the stock arm and run the same command with terminal control disabled: +Rerun the Harbor command with these two changes: ```bash -export RUN_DIR=/absolute/path/printed/by/the/stock/stack/terminal - -harbor run \ - --dataset swebench-verified@1.0 \ - --include-task-name astropy__astropy-12907 \ - --agent harbor_dynamo_pi:DynamoPi \ - --model "dynamo/$PUBLIC_MODEL" \ - --agent-kwarg version=0.72.1 \ - --agent-env "DYNAMO_BASE_URL=$DYNAMO_BASE_URL" \ - --agent-env DYNAMO_API_KEY=dynamo-local \ - --agent-env DYN_AGENT_SESSION_FINAL=0 \ - --mounts "[{\"type\":\"bind\",\"source\":\"$PI_PLUGIN_DIR\",\"target\":\"/opt/pi-dynamo-provider\",\"read_only\":true}]" \ - --n-concurrent 1 \ - --n-concurrent-agents 1 \ - --max-retries 0 \ - --agent-setup-timeout-multiplier 10 \ - --no-force-build \ - --no-delete \ - --environment-kwarg keep_containers=true \ - --jobs-dir "$RUN_DIR/harbor" \ - --job-name kv-verified-one \ - --yes - -find "$RUN_DIR/harbor/kv-verified-one" -name pi.txt -o -name results.json +--ae DYN_AGENT_SESSION_FINAL=0 --job-name kv-verified-one ``` -The two Harbor commands differ only in the job name and `DYN_AGENT_SESSION_FINAL`. Stable session headers are still sent in the stock arm; they are identity metadata and do not enable ThunderAgent or sticky routing. +Stable session headers remain enabled in the stock arm; they are identity metadata and do not enable ThunderAgent or sticky routing. Only the terminal lifecycle request is disabled. + +### 4. Scale the comparison -### 6. Scale the comparison +After both smokes pass, use the same frozen task list and concurrency for both arms. For an exploratory run, replace `-i astropy__astropy-12907 -n 1` with `--n-tasks 32 -n 32 --n-concurrent-agents 32`. For a reported result, use repeated `-i ` arguments from a checked-in manifest because registry order is not a stable cohort. -After both one-task smokes pass, use the same frozen task list and concurrency for both arms. For an exploratory 32-task run, replace `--include-task-name ...` with `--n-tasks 32` and set both concurrency options to 32. For a reported result, use repeated `--include-task-name ` arguments from a checked-in manifest because registry order is not a stable cohort. Add `--disable-verification` only when measuring serving throughput rather than solve rate. +For high concurrency, also add `--no-delete --environment-kwarg keep_containers=true`; inspect and remove the stopped task containers between arms. Add `--disable-verification` only when measuring serving throughput rather than solve rate. Run each arm from fresh server processes. -Start Tachometer before each Harbor run if you need server-side metrics: +Start Tachometer before Harbor when server-side metrics are needed: ```bash tachometer-scraper \ @@ -390,15 +192,7 @@ tachometer-scraper \ --storage "$RUN_DIR/tachometer" ``` -Add `--endpoint thunderagent=http://127.0.0.1:8183/metrics` for the ThunderAgent arm. Run each arm from fresh server processes, pre-pull task images before the measured interval, and preserve the Harbor job directory and server logs. Stop the stock processes when finished: - -```bash -kill "$FRONTEND_PID" "$WORKER_0_PID" "$WORKER_1_PID" -wait "$FRONTEND_PID" "$WORKER_0_PID" "$WORKER_1_PID" 2>/dev/null || true -docker ps -a --filter status=exited -``` - -Inspect the stopped Harbor containers before removing the ones belonging to the completed jobs. +Add `--endpoint thunderagent=http://127.0.0.1:8183/metrics` for the ThunderAgent arm. Set `RUN_DIR` to the desired artifact directory before launching Tachometer. ## Citation diff --git a/components/src/dynamo/thunderagent_router/run_minimax_8xh100.sh b/components/src/dynamo/thunderagent_router/run_minimax_8xh100.sh index 10b0e90a36a5..a63832c95ab9 100755 --- a/components/src/dynamo/thunderagent_router/run_minimax_8xh100.sh +++ b/components/src/dynamo/thunderagent_router/run_minimax_8xh100.sh @@ -2,46 +2,87 @@ # SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 # -# Bring up the thunderagent_router MiniMax-M2 eval stack on a single 8xH100 -# node: two TP4 vLLM workers (GPUs 0-3 and 4-7) + the program-aware router + -# the frontend on :8100. Hardcoded for that one config -- nothing else. -# -# Pair it with the mini-SWE-agent client (see README "Reproducing the MiniMax-M2 -# results"). First launch JIT-warms the FP8 kernels and can take several -# minutes before /v1/models responds. +# Bring up the MiniMax-M2 A/B stack on one 8xH100 node. Usage: +# ./run_minimax_8xh100.sh ta +# ./run_minimax_8xh100.sh kv set -euo pipefail -trap 'echo "Cleaning up..."; kill 0' EXIT + +POLICY="${1:-}" +if [[ "$POLICY" != "ta" && "$POLICY" != "kv" ]]; then + echo "usage: $0 ta|kv" >&2 + exit 2 +fi + +cleanup() { + jobs -pr | xargs -r kill +} +trap cleanup EXIT MODEL="MiniMaxAI/MiniMax-M2" +WORKER_MODEL="$MODEL" BLOCK_SIZE=16 HTTP_PORT=8100 -export PYTHONHASHSEED=0 # deterministic KV-event block hashes +if [[ "$POLICY" == "ta" ]]; then + WORKER_MODEL="dyn-internal-minimax-m2" +fi -# Worker 0: GPUs 0-3. -DYN_SYSTEM_PORT=8081 CUDA_VISIBLE_DEVICES=0,1,2,3 python -m dynamo.vllm \ - --model "$MODEL" --tensor-parallel-size 4 --block-size "$BLOCK_SIZE" \ +export PYTHONHASHSEED=0 +export DYN_DISCOVERY_BACKEND=file +export DYN_FILE_KV="${DYN_FILE_KV:-/tmp/dynamo-minimax-${POLICY}-$$}" +export DYN_REQUEST_PLANE=tcp +export DYN_EVENT_PLANE=zmq +mkdir -p "$DYN_FILE_KV" + +DYN_SYSTEM_PORT=8181 DYN_FORWARDPASS_METRIC_PORT=20081 \ +VLLM_NIXL_SIDE_CHANNEL_PORT=20097 CUDA_VISIBLE_DEVICES=0,1,2,3 \ +python -m dynamo.vllm \ + --model "$MODEL" --served-model-name "$WORKER_MODEL" \ + --tensor-parallel-size 4 --block-size "$BLOCK_SIZE" \ + --kv-cache-dtype fp8 --enable-prefix-caching \ --dyn-tool-call-parser minimax_m2 \ + --dyn-reasoning-parser minimax_append_think \ --kv-events-config '{"publisher":"zmq","topic":"kv-events","endpoint":"tcp://*:20080","enable_kv_cache_events":true}' & -# Worker 1: GPUs 4-7. -DYN_SYSTEM_PORT=8082 CUDA_VISIBLE_DEVICES=4,5,6,7 python -m dynamo.vllm \ - --model "$MODEL" --tensor-parallel-size 4 --block-size "$BLOCK_SIZE" \ +DYN_SYSTEM_PORT=8182 DYN_FORWARDPASS_METRIC_PORT=20082 \ +VLLM_NIXL_SIDE_CHANNEL_PORT=20098 CUDA_VISIBLE_DEVICES=4,5,6,7 \ +python -m dynamo.vllm \ + --model "$MODEL" --served-model-name "$WORKER_MODEL" \ + --tensor-parallel-size 4 --block-size "$BLOCK_SIZE" \ + --kv-cache-dtype fp8 --enable-prefix-caching \ --dyn-tool-call-parser minimax_m2 \ - --kv-events-config '{"publisher":"zmq","topic":"kv-events","endpoint":"tcp://*:20081","enable_kv_cache_events":true}' & + --dyn-reasoning-parser minimax_append_think \ + --kv-events-config '{"publisher":"zmq","topic":"kv-events","endpoint":"tcp://*:20090","enable_kv_cache_events":true}' & -# Program-aware router: registers the model handler and forwards the parser so -# MiniMax's XML reaches the agent as OpenAI tool_calls. -python -m dynamo.thunderagent_router \ - --endpoint dynamo.vllm.generate \ - --model-name "$MODEL" \ - --dyn-tool-call-parser minimax_m2 \ - --router-block-size "$BLOCK_SIZE" & +if [[ "$POLICY" == "ta" ]]; then + DYN_SYSTEM_PORT=8183 python -m dynamo.thunderagent_router \ + --endpoint dynamo.backend.generate \ + --model-name "$MODEL" \ + --dyn-tool-call-parser minimax_m2 \ + --dyn-reasoning-parser minimax_append_think \ + --router-block-size "$BLOCK_SIZE" \ + --router-reset-states \ + --shared-cache-type none & + ROUTER_MODE=round-robin +else + ROUTER_MODE=kv +fi -# Frontend (round-robin; the router owns scheduling and registered the model). -python -m dynamo.frontend \ +DYN_SYSTEM_PORT=8184 python -m dynamo.frontend \ + --http-host 0.0.0.0 \ --http-port "$HTTP_PORT" \ - --router-mode round-robin \ - --router-reset-states & + --router-mode "$ROUTER_MODE" \ + --router-reset-states \ + --shared-cache-type none \ + --admission-control none & + +until curl -fsS "http://127.0.0.1:${HTTP_PORT}/v1/models/${WORKER_MODEL}/ready" 2>/dev/null \ + | jq -e '([.namespaces[].worker_types.aggregated.workers // 0] | add) == 2' >/dev/null; do + sleep 5 +done +until curl -fsS "http://127.0.0.1:${HTTP_PORT}/v1/models" 2>/dev/null | grep -Fq "$MODEL"; do + sleep 5 +done +echo "$POLICY stack ready at http://127.0.0.1:${HTTP_PORT}/v1" wait From 4bccbac7807845eb58c0862dd3beb1ac2fe960c3 Mon Sep 17 00:00:00 2001 From: Ishan Dhanani Date: Sat, 4 Jul 2026 00:12:47 +0000 Subject: [PATCH 10/30] docs(thunderagent): focus Harbor benchmark steps Signed-off-by: Ishan Dhanani --- .../src/dynamo/thunderagent_router/README.md | 68 +++++++++---------- 1 file changed, 33 insertions(+), 35 deletions(-) diff --git a/components/src/dynamo/thunderagent_router/README.md b/components/src/dynamo/thunderagent_router/README.md index 1ff749a46895..3cfc2a4635e7 100644 --- a/components/src/dynamo/thunderagent_router/README.md +++ b/components/src/dynamo/thunderagent_router/README.md @@ -125,22 +125,35 @@ uv sync These commands were validated with Harbor `0.16.0` and Pi `0.72.1`. Record the Dynamo, Harbor, and agent-plugins revisions with benchmark artifacts. -### 2. Run the ThunderAgent smoke +### 2. Launch ThunderAgent or stock KV -The checked-in launcher starts both TP4 workers, ThunderAgent, and the frontend. It prints `ta stack ready` only after both workers register. +Choose one arm. The launcher starts both TP4 workers and the matching router, then prints readiness after both workers and the public model register. ```bash cd ~/src/dynamo source .venv/bin/activate -./components/src/dynamo/thunderagent_router/run_minimax_8xh100.sh ta + +# ThunderAgent +export ARM=ta SESSION_FINAL=1 + +# Stock KV instead +# export ARM=kv SESSION_FINAL=0 + +./components/src/dynamo/thunderagent_router/run_minimax_8xh100.sh "$ARM" ``` -In a second terminal, run one Verified task. Use a host address reachable from Docker rather than `127.0.0.1`. +Stop the launcher with `Ctrl-C` before starting the other arm. Always use fresh server processes for each arm. + +### 3. Run one Verified task + +In a second terminal, set `ARM` and `SESSION_FINAL` to match the launched stack. Use a host address reachable from Docker rather than `127.0.0.1`. ```bash cd ~/src/harbor source .venv/bin/activate +# Match the launched arm. Use ARM=kv SESSION_FINAL=0 for stock KV. +export ARM=ta SESSION_FINAL=1 export PI_PLUGIN_DIR=~/src/agent-plugins/pi-plugin export PYTHONPATH=$PI_PLUGIN_DIR export DYNAMO_BASE_URL=http://$(ip route get 1.1.1.1 | awk '{print $7; exit}'):8100/v1 @@ -150,49 +163,34 @@ harbor run \ -d swebench-verified@1.0 -i astropy__astropy-12907 \ -a harbor_dynamo_pi:DynamoPi -m dynamo/MiniMaxAI/MiniMax-M2 \ --ak version=0.72.1 --ae "DYNAMO_BASE_URL=$DYNAMO_BASE_URL" \ + --ae "DYN_AGENT_SESSION_FINAL=$SESSION_FINAL" \ --mounts "[{\"type\":\"bind\",\"source\":\"$PI_PLUGIN_DIR\",\"target\":\"/opt/pi-dynamo-provider\",\"read_only\":true}]" \ -n 1 --agent-setup-timeout-multiplier 10 \ - --job-name ta-verified-one -y + --job-name "$ARM-verified-one" -y ``` No pause/resume lines are expected from a one-task smoke; those only appear after the working set reaches the configured thresholds. -### 3. Run the stock KV smoke - -Stop the ThunderAgent launcher with `Ctrl-C`, then start a fresh stock-KV stack: - -```bash -cd ~/src/dynamo -source .venv/bin/activate -./components/src/dynamo/thunderagent_router/run_minimax_8xh100.sh kv -``` +### 4. Run the full Verified dataset -Rerun the Harbor command with these two changes: +On a fresh host, first run the command below with `--install-only` and without `--no-delete --environment-kwarg keep_containers=true`. This pre-pulls the task images without issuing model requests; keep it outside the measured interval. ```bash ---ae DYN_AGENT_SESSION_FINAL=0 --job-name kv-verified-one +harbor run \ + -d swebench-verified@1.0 \ + -a harbor_dynamo_pi:DynamoPi -m dynamo/MiniMaxAI/MiniMax-M2 \ + --ak version=0.72.1 --ae "DYNAMO_BASE_URL=$DYNAMO_BASE_URL" \ + --ae "DYN_AGENT_SESSION_FINAL=$SESSION_FINAL" \ + --mounts "[{\"type\":\"bind\",\"source\":\"$PI_PLUGIN_DIR\",\"target\":\"/opt/pi-dynamo-provider\",\"read_only\":true}]" \ + -n 256 --n-concurrent-agents 256 \ + --agent-setup-timeout-multiplier 10 --memory ignore \ + --no-delete --environment-kwarg keep_containers=true \ + --job-name "$ARM-verified-full" -y ``` -Stable session headers remain enabled in the stock arm; they are identity metadata and do not enable ThunderAgent or sticky routing. Only the terminal lifecycle request is disabled. - -### 4. Scale the comparison - -After both smokes pass, use the same frozen task list and concurrency for both arms. For an exploratory run, replace `-i astropy__astropy-12907 -n 1` with `--n-tasks 32 -n 32 --n-concurrent-agents 32`. For a reported result, use repeated `-i ` arguments from a checked-in manifest because registry order is not a stable cohort. - -For high concurrency, also add `--no-delete --environment-kwarg keep_containers=true`; inspect and remove the stopped task containers between arms. Add `--disable-verification` only when measuring serving throughput rather than solve rate. Run each arm from fresh server processes. - -Start Tachometer before Harbor when server-side metrics are needed: - -```bash -tachometer-scraper \ - --endpoint frontend=http://127.0.0.1:8100/metrics \ - --endpoint worker0=http://127.0.0.1:8181/metrics \ - --endpoint worker1=http://127.0.0.1:8182/metrics \ - --freq 1.0 \ - --storage "$RUN_DIR/tachometer" -``` +This runs every task in `swebench-verified@1.0` with verification enabled. Remove the stopped task containers, start the other arm from fresh processes, set its matching `ARM` and `SESSION_FINAL`, and rerun the same command. -Add `--endpoint thunderagent=http://127.0.0.1:8183/metrics` for the ThunderAgent arm. Set `RUN_DIR` to the desired artifact directory before launching Tachometer. +Stable session headers are sent in both arms. `SESSION_FINAL=0` only prevents the stock KV arm from receiving ThunderAgent's terminal lifecycle request. ## Citation From b31bfb2916cd2164a3bb70f2930f06fb781373a3 Mon Sep 17 00:00:00 2001 From: Ishan Dhanani Date: Sat, 4 Jul 2026 00:26:47 +0000 Subject: [PATCH 11/30] fix(thunderagent): pin reproducible MiniMax setup Signed-off-by: Ishan Dhanani --- .../src/dynamo/thunderagent_router/README.md | 8 ++++++-- .../thunderagent_router/run_minimax_8xh100.sh | 14 ++++++++------ 2 files changed, 14 insertions(+), 8 deletions(-) diff --git a/components/src/dynamo/thunderagent_router/README.md b/components/src/dynamo/thunderagent_router/README.md index 3cfc2a4635e7..d6d26ee46144 100644 --- a/components/src/dynamo/thunderagent_router/README.md +++ b/components/src/dynamo/thunderagent_router/README.md @@ -98,7 +98,7 @@ agent. ## Harbor/Pi A/B walkthrough -This walkthrough runs the same SWE-bench Verified task through ThunderAgent and the stock Dynamo KV router. Harbor owns the task container, Pi runs inside it, and the model stack runs on the host. The example uses one 8-GPU node, two TP4 vLLM workers, and MiniMax-M2. There is no HiCache, Mooncake, shared cache, or frontend admission control in either arm. +This walkthrough runs the same SWE-bench Verified task through ThunderAgent and the stock Dynamo KV router. Harbor owns the task container, Pi runs inside it, and the model stack runs on the host. The example uses one 8-GPU node and two TP4 vLLM workers loading `MiniMaxAI/MiniMax-M2.7` from Hugging Face and serving the API alias `MiniMaxAI/MiniMax-M2`. There is no HiCache, Mooncake, shared cache, or frontend admission control in either arm. The [agent-plugins `DynamoPi` adapter](https://github.com/ai-dynamo/agent-plugins/blob/main/pi-plugin/harbor_dynamo_pi.py) is required. It installs the Dynamo provider in each Harbor task container and maps Harbor's per-trial ID to one stable `x-dynamo-session-id` across all Pi turns. The ThunderAgent arm also sends one terminal session request so the router can release the completed program; the stock KV arm disables that request because it has no lifecycle consumer. @@ -108,8 +108,12 @@ Use Python 3.12 and a machine with Docker, Node.js, and eight visible GPUs. Buil ```bash git clone https://github.com/ai-dynamo/dynamo ~/src/dynamo -git clone https://github.com/harbor-framework/harbor ~/src/harbor +git -C ~/src/dynamo fetch origin pull/11185/head +git -C ~/src/dynamo checkout --detach FETCH_HEAD + +git clone --branch v0.16.0 --depth 1 https://github.com/harbor-framework/harbor ~/src/harbor git clone https://github.com/ai-dynamo/agent-plugins ~/src/agent-plugins +git -C ~/src/agent-plugins checkout d0bae6a3c486c937f1b190ce21259a828cbaa8f9 cd ~/src/dynamo uv venv --python 3.12 --seed .venv diff --git a/components/src/dynamo/thunderagent_router/run_minimax_8xh100.sh b/components/src/dynamo/thunderagent_router/run_minimax_8xh100.sh index a63832c95ab9..d4bf8d152ea7 100755 --- a/components/src/dynamo/thunderagent_router/run_minimax_8xh100.sh +++ b/components/src/dynamo/thunderagent_router/run_minimax_8xh100.sh @@ -18,8 +18,9 @@ cleanup() { } trap cleanup EXIT -MODEL="MiniMaxAI/MiniMax-M2" -WORKER_MODEL="$MODEL" +MODEL_PATH="${MODEL_PATH:-MiniMaxAI/MiniMax-M2.7}" +MODEL_NAME="${MODEL_NAME:-MiniMaxAI/MiniMax-M2}" +WORKER_MODEL="$MODEL_NAME" BLOCK_SIZE=16 HTTP_PORT=8100 @@ -37,7 +38,7 @@ mkdir -p "$DYN_FILE_KV" DYN_SYSTEM_PORT=8181 DYN_FORWARDPASS_METRIC_PORT=20081 \ VLLM_NIXL_SIDE_CHANNEL_PORT=20097 CUDA_VISIBLE_DEVICES=0,1,2,3 \ python -m dynamo.vllm \ - --model "$MODEL" --served-model-name "$WORKER_MODEL" \ + --model "$MODEL_PATH" --served-model-name "$WORKER_MODEL" \ --tensor-parallel-size 4 --block-size "$BLOCK_SIZE" \ --kv-cache-dtype fp8 --enable-prefix-caching \ --dyn-tool-call-parser minimax_m2 \ @@ -47,7 +48,7 @@ python -m dynamo.vllm \ DYN_SYSTEM_PORT=8182 DYN_FORWARDPASS_METRIC_PORT=20082 \ VLLM_NIXL_SIDE_CHANNEL_PORT=20098 CUDA_VISIBLE_DEVICES=4,5,6,7 \ python -m dynamo.vllm \ - --model "$MODEL" --served-model-name "$WORKER_MODEL" \ + --model "$MODEL_PATH" --served-model-name "$WORKER_MODEL" \ --tensor-parallel-size 4 --block-size "$BLOCK_SIZE" \ --kv-cache-dtype fp8 --enable-prefix-caching \ --dyn-tool-call-parser minimax_m2 \ @@ -57,7 +58,8 @@ python -m dynamo.vllm \ if [[ "$POLICY" == "ta" ]]; then DYN_SYSTEM_PORT=8183 python -m dynamo.thunderagent_router \ --endpoint dynamo.backend.generate \ - --model-name "$MODEL" \ + --model-name "$MODEL_NAME" \ + --model-path "$MODEL_PATH" \ --dyn-tool-call-parser minimax_m2 \ --dyn-reasoning-parser minimax_append_think \ --router-block-size "$BLOCK_SIZE" \ @@ -80,7 +82,7 @@ until curl -fsS "http://127.0.0.1:${HTTP_PORT}/v1/models/${WORKER_MODEL}/ready" | jq -e '([.namespaces[].worker_types.aggregated.workers // 0] | add) == 2' >/dev/null; do sleep 5 done -until curl -fsS "http://127.0.0.1:${HTTP_PORT}/v1/models" 2>/dev/null | grep -Fq "$MODEL"; do +until curl -fsS "http://127.0.0.1:${HTTP_PORT}/v1/models" 2>/dev/null | grep -Fq "$MODEL_NAME"; do sleep 5 done echo "$POLICY stack ready at http://127.0.0.1:${HTTP_PORT}/v1" From 81c4948bec5e47b1f901a368fb94d0642aad54fc Mon Sep 17 00:00:00 2001 From: Ishan Dhanani Date: Sat, 4 Jul 2026 00:32:27 +0000 Subject: [PATCH 12/30] docs(thunderagent): use cached Hugging Face model Signed-off-by: Ishan Dhanani --- components/src/dynamo/thunderagent_router/README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/components/src/dynamo/thunderagent_router/README.md b/components/src/dynamo/thunderagent_router/README.md index d6d26ee46144..65006a94ea60 100644 --- a/components/src/dynamo/thunderagent_router/README.md +++ b/components/src/dynamo/thunderagent_router/README.md @@ -136,6 +136,7 @@ Choose one arm. The launcher starts both TP4 workers and the matching router, th ```bash cd ~/src/dynamo source .venv/bin/activate +export HF_HOME=/home/nvidia/hf_cache # ThunderAgent export ARM=ta SESSION_FINAL=1 From afd3f4953613153b256c3e5ac717692617eec811 Mon Sep 17 00:00:00 2001 From: Ishan Dhanani Date: Sat, 4 Jul 2026 01:46:02 +0000 Subject: [PATCH 13/30] docs(thunderagent): make Harbor scale setup reliable Signed-off-by: Ishan Dhanani --- components/src/dynamo/thunderagent_router/README.md | 5 +++-- .../src/dynamo/thunderagent_router/harbor-host-network.yml | 3 +++ 2 files changed, 6 insertions(+), 2 deletions(-) create mode 100644 components/src/dynamo/thunderagent_router/harbor-host-network.yml diff --git a/components/src/dynamo/thunderagent_router/README.md b/components/src/dynamo/thunderagent_router/README.md index 65006a94ea60..c53c8f14d523 100644 --- a/components/src/dynamo/thunderagent_router/README.md +++ b/components/src/dynamo/thunderagent_router/README.md @@ -104,7 +104,7 @@ The [agent-plugins `DynamoPi` adapter](https://github.com/ai-dynamo/agent-plugin ### 1. Install the three source trees -Use Python 3.12 and a machine with Docker, Node.js, and eight visible GPUs. Build Dynamo from the branch under test rather than installing a released wheel. +Use Python 3.12 and a machine with Docker and eight visible GPUs. Build Dynamo from the branch under test rather than installing a released wheel. Pi and its Node.js runtime are installed inside each Harbor task container. ```bash git clone https://github.com/ai-dynamo/dynamo ~/src/dynamo @@ -178,7 +178,7 @@ No pause/resume lines are expected from a one-task smoke; those only appear afte ### 4. Run the full Verified dataset -On a fresh host, first run the command below with `--install-only` and without `--no-delete --environment-kwarg keep_containers=true`. This pre-pulls the task images without issuing model requests; keep it outside the measured interval. +On a fresh host, first run the command below with `--install-only`, change both concurrency values from `256` to `32`, and omit `--no-delete --environment-kwarg keep_containers=true`. This pre-pulls the task images without issuing model requests; keep it outside the measured interval. The lower install concurrency avoids overwhelming Docker during simultaneous container teardown. The host-network overlay avoids creating one Docker bridge network per trial, which exhausts Docker's default address pools during the measured run. ```bash harbor run \ @@ -187,6 +187,7 @@ harbor run \ --ak version=0.72.1 --ae "DYNAMO_BASE_URL=$DYNAMO_BASE_URL" \ --ae "DYN_AGENT_SESSION_FINAL=$SESSION_FINAL" \ --mounts "[{\"type\":\"bind\",\"source\":\"$PI_PLUGIN_DIR\",\"target\":\"/opt/pi-dynamo-provider\",\"read_only\":true}]" \ + --extra-docker-compose ~/src/dynamo/components/src/dynamo/thunderagent_router/harbor-host-network.yml \ -n 256 --n-concurrent-agents 256 \ --agent-setup-timeout-multiplier 10 --memory ignore \ --no-delete --environment-kwarg keep_containers=true \ diff --git a/components/src/dynamo/thunderagent_router/harbor-host-network.yml b/components/src/dynamo/thunderagent_router/harbor-host-network.yml new file mode 100644 index 000000000000..0157f393efcc --- /dev/null +++ b/components/src/dynamo/thunderagent_router/harbor-host-network.yml @@ -0,0 +1,3 @@ +services: + main: + network_mode: host From b7e42169a10eb18277d36d545e6924129c1390b6 Mon Sep 17 00:00:00 2001 From: Ishan Dhanani Date: Sat, 4 Jul 2026 01:52:45 +0000 Subject: [PATCH 14/30] fix(thunderagent): satisfy repro checks --- .../src/dynamo/sglang/tests/test_runtime_metadata.py | 8 +++++--- .../dynamo/thunderagent_router/harbor-host-network.yml | 3 +++ 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/components/src/dynamo/sglang/tests/test_runtime_metadata.py b/components/src/dynamo/sglang/tests/test_runtime_metadata.py index 8f7f54b2aa53..f8757de81361 100644 --- a/components/src/dynamo/sglang/tests/test_runtime_metadata.py +++ b/components/src/dynamo/sglang/tests/test_runtime_metadata.py @@ -6,7 +6,6 @@ import pytest from dynamo.sglang.capacity import get_spec_decode_runtime_data -from dynamo.sglang.register import _get_hicache_capacity_runtime_data pytestmark = [ pytest.mark.unit, @@ -74,6 +73,8 @@ def test_eagle_enabled_for_speculative_algorithm(speculative_algorithm, expected def test_hicache_capacity_runtime_data_uses_scheduler_capacity(): + from dynamo.sglang.register import _get_hicache_capacity_runtime_data + assert _get_hicache_capacity_runtime_data({"hicache_host_total_tokens": 300}) == { "host_total_tokens": 300 } @@ -81,7 +82,8 @@ def test_hicache_capacity_runtime_data_uses_scheduler_capacity(): @pytest.mark.parametrize("value", [None, 0, -1, "300"]) def test_hicache_capacity_runtime_data_ignores_invalid_values(value): + from dynamo.sglang.register import _get_hicache_capacity_runtime_data + assert ( - _get_hicache_capacity_runtime_data({"hicache_host_total_tokens": value}) - is None + _get_hicache_capacity_runtime_data({"hicache_host_total_tokens": value}) is None ) diff --git a/components/src/dynamo/thunderagent_router/harbor-host-network.yml b/components/src/dynamo/thunderagent_router/harbor-host-network.yml index 0157f393efcc..2187d40fbd02 100644 --- a/components/src/dynamo/thunderagent_router/harbor-host-network.yml +++ b/components/src/dynamo/thunderagent_router/harbor-host-network.yml @@ -1,3 +1,6 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + services: main: network_mode: host From 97d7ac5315cf960b9c2f3f23f42310f8506d9deb Mon Sep 17 00:00:00 2001 From: Ishan Dhanani Date: Sat, 4 Jul 2026 01:56:38 +0000 Subject: [PATCH 15/30] docs(thunderagent): require Docker Hub login --- components/src/dynamo/thunderagent_router/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/src/dynamo/thunderagent_router/README.md b/components/src/dynamo/thunderagent_router/README.md index c53c8f14d523..d4b040a714e9 100644 --- a/components/src/dynamo/thunderagent_router/README.md +++ b/components/src/dynamo/thunderagent_router/README.md @@ -178,7 +178,7 @@ No pause/resume lines are expected from a one-task smoke; those only appear afte ### 4. Run the full Verified dataset -On a fresh host, first run the command below with `--install-only`, change both concurrency values from `256` to `32`, and omit `--no-delete --environment-kwarg keep_containers=true`. This pre-pulls the task images without issuing model requests; keep it outside the measured interval. The lower install concurrency avoids overwhelming Docker during simultaneous container teardown. The host-network overlay avoids creating one Docker bridge network per trial, which exhausts Docker's default address pools during the measured run. +On a fresh host, authenticate with Docker Hub using `docker login`; the 500 unique Verified task images exceed the anonymous pull limit. Then run the command below with `--install-only`, change both concurrency values from `256` to `32`, and omit `--no-delete --environment-kwarg keep_containers=true`. This pre-pulls the task images without issuing model requests; keep it outside the measured interval. The lower install concurrency avoids overwhelming Docker during simultaneous container teardown. The host-network overlay avoids creating one Docker bridge network per trial, which exhausts Docker's default address pools during the measured run. ```bash harbor run \ From 5a17e2236df9f4c59ed9a3b07bd87324924dc055 Mon Sep 17 00:00:00 2001 From: Ishan Dhanani Date: Sat, 4 Jul 2026 01:58:55 +0000 Subject: [PATCH 16/30] docs(thunderagent): fix online repro link --- docs/agents/thunderagent-router.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/agents/thunderagent-router.md b/docs/agents/thunderagent-router.md index 6a54a7c77590..3148ab76d4c1 100644 --- a/docs/agents/thunderagent-router.md +++ b/docs/agents/thunderagent-router.md @@ -128,7 +128,7 @@ For per-request tracing (token counts, cache hits, worker placement), the router ## Reproducing with upstream Harbor and Pi -The maintained end-to-end path uses upstream Harbor to create SWE-bench containers and Pi with the Dynamo provider inside each container. The complete source build, ThunderAgent arm, stock KV arm, stable-session setup, and scaling procedure live next to the experimental component in [`components/src/dynamo/thunderagent_router/README.md`](/components/src/dynamo/thunderagent_router/README.md#harborpi-ab-walkthrough). +The maintained end-to-end path uses upstream Harbor to create SWE-bench containers and Pi with the Dynamo provider inside each container. The complete source build, ThunderAgent arm, stock KV arm, stable-session setup, and scaling procedure live in the [`thunderagent_router` README](https://github.com/ai-dynamo/dynamo/blob/main/components/src/dynamo/thunderagent_router/README.md#harborpi-ab-walkthrough). ## References From 37bf8b15bd6f58d0774ed41e5c34fae2ab64c86f Mon Sep 17 00:00:00 2001 From: Ishan Dhanani Date: Sat, 4 Jul 2026 09:57:19 +0000 Subject: [PATCH 17/30] docs(thunderagent): add reliable image pre-pull --- .../src/dynamo/thunderagent_router/README.md | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/components/src/dynamo/thunderagent_router/README.md b/components/src/dynamo/thunderagent_router/README.md index d4b040a714e9..4721e88aac16 100644 --- a/components/src/dynamo/thunderagent_router/README.md +++ b/components/src/dynamo/thunderagent_router/README.md @@ -178,7 +178,24 @@ No pause/resume lines are expected from a one-task smoke; those only appear afte ### 4. Run the full Verified dataset -On a fresh host, authenticate with Docker Hub using `docker login`; the 500 unique Verified task images exceed the anonymous pull limit. Then run the command below with `--install-only`, change both concurrency values from `256` to `32`, and omit `--no-delete --environment-kwarg keep_containers=true`. This pre-pulls the task images without issuing model requests; keep it outside the measured interval. The lower install concurrency avoids overwhelming Docker during simultaneous container teardown. The host-network overlay avoids creating one Docker bridge network per trial, which exhausts Docker's default address pools during the measured run. +On a fresh host, authenticate with Docker Hub and prepare every task image outside the measured interval. Verified uses 500 unique images, which exceeds the anonymous pull limit. Keeping containers until the job finishes avoids hundreds of concurrent per-trial Compose teardowns; prune them once afterward. + +```bash +docker login + +harbor run \ + -d swebench-verified@1.0 \ + -a nop \ + --extra-docker-compose ~/src/dynamo/components/src/dynamo/thunderagent_router/harbor-host-network.yml \ + -n 32 --n-concurrent-agents 32 \ + --memory ignore \ + --no-delete --environment-kwarg keep_containers=true \ + --job-name verified-prepull --install-only -y + +docker container prune -f +``` + +The host-network overlay avoids creating one Docker bridge network per trial, which exhausts Docker's default address pools during the measured run. ```bash harbor run \ From 4cabe9fa7daee10a430fb7832775d63f660ee940 Mon Sep 17 00:00:00 2001 From: Ishan Dhanani Date: Sat, 4 Jul 2026 10:45:42 +0000 Subject: [PATCH 18/30] docs(thunderagent): address Harbor walkthrough review Signed-off-by: Ishan Dhanani --- .../src/dynamo/thunderagent_router/README.md | 24 +++++++++---------- 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/components/src/dynamo/thunderagent_router/README.md b/components/src/dynamo/thunderagent_router/README.md index 4721e88aac16..c0772d11deac 100644 --- a/components/src/dynamo/thunderagent_router/README.md +++ b/components/src/dynamo/thunderagent_router/README.md @@ -98,7 +98,7 @@ agent. ## Harbor/Pi A/B walkthrough -This walkthrough runs the same SWE-bench Verified task through ThunderAgent and the stock Dynamo KV router. Harbor owns the task container, Pi runs inside it, and the model stack runs on the host. The example uses one 8-GPU node and two TP4 vLLM workers loading `MiniMaxAI/MiniMax-M2.7` from Hugging Face and serving the API alias `MiniMaxAI/MiniMax-M2`. There is no HiCache, Mooncake, shared cache, or frontend admission control in either arm. +This walkthrough runs the same SWE-bench Verified task through ThunderAgent and the stock Dynamo KV router. Harbor owns the task container, Pi runs inside it, and the model stack runs on the host. ThunderAgent is backend-agnostic and works with Dynamo's vLLM and SGLang backends; this example uses one 8-GPU node and two TP4 vLLM workers loading `MiniMaxAI/MiniMax-M2.7` from Hugging Face and serving the API alias `MiniMaxAI/MiniMax-M2`. There is no HiCache, Mooncake, shared cache, or frontend admission control in either arm. The [agent-plugins `DynamoPi` adapter](https://github.com/ai-dynamo/agent-plugins/blob/main/pi-plugin/harbor_dynamo_pi.py) is required. It installs the Dynamo provider in each Harbor task container and maps Harbor's per-trial ID to one stable `x-dynamo-session-id` across all Pi turns. The ThunderAgent arm also sends one terminal session request so the router can release the completed program; the stock KV arm disables that request because it has no lifecycle consumer. @@ -120,7 +120,6 @@ uv venv --python 3.12 --seed .venv source .venv/bin/activate uv pip install pip 'maturin[patchelf]' (cd lib/bindings/python && maturin develop --uv) -uv pip install -e lib/gpu_memory_service uv pip install -e '.[vllm]' cd ~/src/harbor @@ -139,10 +138,10 @@ source .venv/bin/activate export HF_HOME=/home/nvidia/hf_cache # ThunderAgent -export ARM=ta SESSION_FINAL=1 +export ARM=ta # Stock KV instead -# export ARM=kv SESSION_FINAL=0 +# export ARM=kv ./components/src/dynamo/thunderagent_router/run_minimax_8xh100.sh "$ARM" ``` @@ -151,24 +150,25 @@ Stop the launcher with `Ctrl-C` before starting the other arm. Always use fresh ### 3. Run one Verified task -In a second terminal, set `ARM` and `SESSION_FINAL` to match the launched stack. Use a host address reachable from Docker rather than `127.0.0.1`. +In a second terminal, set `ARM` to match the launched stack. The terminal-session flag is derived from it. Use a host address reachable from Docker rather than `127.0.0.1`. ```bash cd ~/src/harbor source .venv/bin/activate -# Match the launched arm. Use ARM=kv SESSION_FINAL=0 for stock KV. -export ARM=ta SESSION_FINAL=1 +# Match the launched arm. Use ARM=kv for stock KV. +export ARM=ta +export DYN_AGENT_SESSION_FINAL=$([[ "$ARM" == ta ]] && echo 1 || echo 0) export PI_PLUGIN_DIR=~/src/agent-plugins/pi-plugin export PYTHONPATH=$PI_PLUGIN_DIR export DYNAMO_BASE_URL=http://$(ip route get 1.1.1.1 | awk '{print $7; exit}'):8100/v1 -docker run --rm curlimages/curl:8.11.1 -fsS "$DYNAMO_BASE_URL/models" +curl -fsS "$DYNAMO_BASE_URL/models" harbor run \ -d swebench-verified@1.0 -i astropy__astropy-12907 \ -a harbor_dynamo_pi:DynamoPi -m dynamo/MiniMaxAI/MiniMax-M2 \ --ak version=0.72.1 --ae "DYNAMO_BASE_URL=$DYNAMO_BASE_URL" \ - --ae "DYN_AGENT_SESSION_FINAL=$SESSION_FINAL" \ + --ae "DYN_AGENT_SESSION_FINAL=$DYN_AGENT_SESSION_FINAL" \ --mounts "[{\"type\":\"bind\",\"source\":\"$PI_PLUGIN_DIR\",\"target\":\"/opt/pi-dynamo-provider\",\"read_only\":true}]" \ -n 1 --agent-setup-timeout-multiplier 10 \ --job-name "$ARM-verified-one" -y @@ -202,7 +202,7 @@ harbor run \ -d swebench-verified@1.0 \ -a harbor_dynamo_pi:DynamoPi -m dynamo/MiniMaxAI/MiniMax-M2 \ --ak version=0.72.1 --ae "DYNAMO_BASE_URL=$DYNAMO_BASE_URL" \ - --ae "DYN_AGENT_SESSION_FINAL=$SESSION_FINAL" \ + --ae "DYN_AGENT_SESSION_FINAL=$DYN_AGENT_SESSION_FINAL" \ --mounts "[{\"type\":\"bind\",\"source\":\"$PI_PLUGIN_DIR\",\"target\":\"/opt/pi-dynamo-provider\",\"read_only\":true}]" \ --extra-docker-compose ~/src/dynamo/components/src/dynamo/thunderagent_router/harbor-host-network.yml \ -n 256 --n-concurrent-agents 256 \ @@ -211,9 +211,9 @@ harbor run \ --job-name "$ARM-verified-full" -y ``` -This runs every task in `swebench-verified@1.0` with verification enabled. Remove the stopped task containers, start the other arm from fresh processes, set its matching `ARM` and `SESSION_FINAL`, and rerun the same command. +This runs every task in `swebench-verified@1.0` with verification enabled. Remove the stopped task containers, start the other arm from fresh processes, set its matching `ARM`, and rerun the same command. -Stable session headers are sent in both arms. `SESSION_FINAL=0` only prevents the stock KV arm from receiving ThunderAgent's terminal lifecycle request. +Stable session headers are sent in both arms. The derived `DYN_AGENT_SESSION_FINAL=0` only prevents the stock KV arm from receiving ThunderAgent's terminal lifecycle request. ## Citation From 3648136520eeb5a03d6bc88b8c669e907b935672 Mon Sep 17 00:00:00 2001 From: Ishan Dhanani Date: Sat, 4 Jul 2026 11:00:06 +0000 Subject: [PATCH 19/30] docs(thunderagent): make session final explicit Signed-off-by: Ishan Dhanani --- .../src/dynamo/thunderagent_router/README.md | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/components/src/dynamo/thunderagent_router/README.md b/components/src/dynamo/thunderagent_router/README.md index c0772d11deac..de34485b62dd 100644 --- a/components/src/dynamo/thunderagent_router/README.md +++ b/components/src/dynamo/thunderagent_router/README.md @@ -150,15 +150,20 @@ Stop the launcher with `Ctrl-C` before starting the other arm. Always use fresh ### 3. Run one Verified task -In a second terminal, set `ARM` to match the launched stack. The terminal-session flag is derived from it. Use a host address reachable from Docker rather than `127.0.0.1`. +In a second terminal, set `ARM` and `DYN_AGENT_SESSION_FINAL` to match the launched stack. Use a host address reachable from Docker rather than `127.0.0.1`. ```bash cd ~/src/harbor source .venv/bin/activate -# Match the launched arm. Use ARM=kv for stock KV. +# ThunderAgent export ARM=ta -export DYN_AGENT_SESSION_FINAL=$([[ "$ARM" == ta ]] && echo 1 || echo 0) +export DYN_AGENT_SESSION_FINAL=1 + +# Stock KV instead +# export ARM=kv +# export DYN_AGENT_SESSION_FINAL=0 + export PI_PLUGIN_DIR=~/src/agent-plugins/pi-plugin export PYTHONPATH=$PI_PLUGIN_DIR export DYNAMO_BASE_URL=http://$(ip route get 1.1.1.1 | awk '{print $7; exit}'):8100/v1 @@ -211,9 +216,9 @@ harbor run \ --job-name "$ARM-verified-full" -y ``` -This runs every task in `swebench-verified@1.0` with verification enabled. Remove the stopped task containers, start the other arm from fresh processes, set its matching `ARM`, and rerun the same command. +This runs every task in `swebench-verified@1.0` with verification enabled. Remove the stopped task containers, start the other arm from fresh processes, set its matching `ARM` and `DYN_AGENT_SESSION_FINAL`, and rerun the same command. -Stable session headers are sent in both arms. The derived `DYN_AGENT_SESSION_FINAL=0` only prevents the stock KV arm from receiving ThunderAgent's terminal lifecycle request. +Stable session headers are sent in both arms. `DYN_AGENT_SESSION_FINAL=0` only prevents the stock KV arm from receiving ThunderAgent's terminal lifecycle request. ## Citation From ed42f0e4907b001713322d56c23e4f462cbca542 Mon Sep 17 00:00:00 2001 From: Ishan Dhanani Date: Sun, 5 Jul 2026 10:41:58 +0000 Subject: [PATCH 20/30] docs(agents): simplify Harbor and Pi setup Signed-off-by: Ishan Dhanani --- components/src/dynamo/thunderagent_router/README.md | 6 +++--- .../src/dynamo/thunderagent_router/harbor-host-network.yml | 6 ------ docs/agents/agent-harnesses.md | 2 +- 3 files changed, 4 insertions(+), 10 deletions(-) delete mode 100644 components/src/dynamo/thunderagent_router/harbor-host-network.yml diff --git a/components/src/dynamo/thunderagent_router/README.md b/components/src/dynamo/thunderagent_router/README.md index de34485b62dd..402905d2d7c1 100644 --- a/components/src/dynamo/thunderagent_router/README.md +++ b/components/src/dynamo/thunderagent_router/README.md @@ -113,7 +113,7 @@ git -C ~/src/dynamo checkout --detach FETCH_HEAD git clone --branch v0.16.0 --depth 1 https://github.com/harbor-framework/harbor ~/src/harbor git clone https://github.com/ai-dynamo/agent-plugins ~/src/agent-plugins -git -C ~/src/agent-plugins checkout d0bae6a3c486c937f1b190ce21259a828cbaa8f9 +git -C ~/src/agent-plugins checkout cd0434136a92522532ee3ae5bf61078f47e6761a cd ~/src/dynamo uv venv --python 3.12 --seed .venv @@ -191,7 +191,7 @@ docker login harbor run \ -d swebench-verified@1.0 \ -a nop \ - --extra-docker-compose ~/src/dynamo/components/src/dynamo/thunderagent_router/harbor-host-network.yml \ + --extra-docker-compose ~/src/agent-plugins/pi-plugin/harbor-host-network.yml \ -n 32 --n-concurrent-agents 32 \ --memory ignore \ --no-delete --environment-kwarg keep_containers=true \ @@ -209,7 +209,7 @@ harbor run \ --ak version=0.72.1 --ae "DYNAMO_BASE_URL=$DYNAMO_BASE_URL" \ --ae "DYN_AGENT_SESSION_FINAL=$DYN_AGENT_SESSION_FINAL" \ --mounts "[{\"type\":\"bind\",\"source\":\"$PI_PLUGIN_DIR\",\"target\":\"/opt/pi-dynamo-provider\",\"read_only\":true}]" \ - --extra-docker-compose ~/src/dynamo/components/src/dynamo/thunderagent_router/harbor-host-network.yml \ + --extra-docker-compose ~/src/agent-plugins/pi-plugin/harbor-host-network.yml \ -n 256 --n-concurrent-agents 256 \ --agent-setup-timeout-multiplier 10 --memory ignore \ --no-delete --environment-kwarg keep_containers=true \ diff --git a/components/src/dynamo/thunderagent_router/harbor-host-network.yml b/components/src/dynamo/thunderagent_router/harbor-host-network.yml deleted file mode 100644 index 2187d40fbd02..000000000000 --- a/components/src/dynamo/thunderagent_router/harbor-host-network.yml +++ /dev/null @@ -1,6 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -services: - main: - network_mode: host diff --git a/docs/agents/agent-harnesses.md b/docs/agents/agent-harnesses.md index a4400acb6029..5c241ee43cd9 100644 --- a/docs/agents/agent-harnesses.md +++ b/docs/agents/agent-harnesses.md @@ -53,7 +53,7 @@ export DYNAMO_BASE_URL=http://localhost:8000/v1 export DYNAMO_API_KEY=dynamo-local export DYN_AGENT_SESSION_ID=pi-demo-$(uuidgen) -pi --model dynamo/zai-org/GLM-4.7-Flash -p 'Reply exactly ok.' +pi --model dynamo/zai-org/GLM-4.7-Flash ``` ## Claude Code From 60de1010bf9af97ce622da3849f5aa695d7c1854 Mon Sep 17 00:00:00 2001 From: Ishan Dhanani Date: Sun, 5 Jul 2026 10:46:39 +0000 Subject: [PATCH 21/30] docs(thunderagent): clone Dynamo main in walkthrough Signed-off-by: Ishan Dhanani --- components/src/dynamo/thunderagent_router/README.md | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/components/src/dynamo/thunderagent_router/README.md b/components/src/dynamo/thunderagent_router/README.md index 402905d2d7c1..d141cc5a2493 100644 --- a/components/src/dynamo/thunderagent_router/README.md +++ b/components/src/dynamo/thunderagent_router/README.md @@ -104,12 +104,10 @@ The [agent-plugins `DynamoPi` adapter](https://github.com/ai-dynamo/agent-plugin ### 1. Install the three source trees -Use Python 3.12 and a machine with Docker and eight visible GPUs. Build Dynamo from the branch under test rather than installing a released wheel. Pi and its Node.js runtime are installed inside each Harbor task container. +Use Python 3.12 and a machine with Docker and eight visible GPUs. Build Dynamo from source rather than installing a released wheel. Pi and its Node.js runtime are installed inside each Harbor task container. ```bash git clone https://github.com/ai-dynamo/dynamo ~/src/dynamo -git -C ~/src/dynamo fetch origin pull/11185/head -git -C ~/src/dynamo checkout --detach FETCH_HEAD git clone --branch v0.16.0 --depth 1 https://github.com/harbor-framework/harbor ~/src/harbor git clone https://github.com/ai-dynamo/agent-plugins ~/src/agent-plugins From a2c7da93c91b39f03eea8749fe7cef65394eec59 Mon Sep 17 00:00:00 2001 From: Ishan Dhanani Date: Sun, 5 Jul 2026 11:14:53 +0000 Subject: [PATCH 22/30] ci(docs): ignore runner-hostile external links Signed-off-by: Ishan Dhanani --- .lycheeignore | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.lycheeignore b/.lycheeignore index 163c3419baa7..daf20a0b730c 100644 --- a/.lycheeignore +++ b/.lycheeignore @@ -21,3 +21,7 @@ # Served by Netlify and intermittently fails from GitHub runners. ^https://etcd\.io/?$ ^https://etcd\.io/docs/v3\.5/install/?$ + +# Unreliable from GitHub runners; these pages remain browser-accessible. +^https://cert-manager\.io/docs/?$ +^https://catalog\.ngc\.nvidia\.com/orgs/nvidia/teams/ai-dynamo/ From da7984b63d1e32523def7a134afc1beb39e5f213 Mon Sep 17 00:00:00 2001 From: Ishan Dhanani Date: Sun, 5 Jul 2026 11:23:48 +0000 Subject: [PATCH 23/30] ci(docs): ignore additional unstable links Signed-off-by: Ishan Dhanani --- .lycheeignore | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.lycheeignore b/.lycheeignore index daf20a0b730c..c4389932b335 100644 --- a/.lycheeignore +++ b/.lycheeignore @@ -25,3 +25,5 @@ # Unreliable from GitHub runners; these pages remain browser-accessible. ^https://cert-manager\.io/docs/?$ ^https://catalog\.ngc\.nvidia\.com/orgs/nvidia/teams/ai-dynamo/ +^https://helm\.sh/docs/intro/install/?$ +^https://book\.kubebuilder\.io/cronjob-tutorial/webhook-implementation\.html$ From 38553e44c451907c59021cd735c68ae1db29b92c Mon Sep 17 00:00:00 2001 From: Ishan Dhanani Date: Sun, 5 Jul 2026 11:33:27 +0000 Subject: [PATCH 24/30] ci(docs): keep PR link checks offline Signed-off-by: Ishan Dhanani --- .github/workflows/docs-link-check.yml | 1 + .lycheeignore | 6 ------ deploy/helm/charts/platform/README.md | 4 ++-- deploy/helm/charts/platform/values.yaml | 4 ++-- 4 files changed, 5 insertions(+), 10 deletions(-) diff --git a/.github/workflows/docs-link-check.yml b/.github/workflows/docs-link-check.yml index 5cb0a21370f8..9ce0af918f74 100644 --- a/.github/workflows/docs-link-check.yml +++ b/.github/workflows/docs-link-check.yml @@ -45,6 +45,7 @@ jobs: --host-concurrency 10 --host-request-interval 1s --host-stats + ${{ github.event_name == 'pull_request' && '--offline' || '' }} . fail: true env: diff --git a/.lycheeignore b/.lycheeignore index c4389932b335..163c3419baa7 100644 --- a/.lycheeignore +++ b/.lycheeignore @@ -21,9 +21,3 @@ # Served by Netlify and intermittently fails from GitHub runners. ^https://etcd\.io/?$ ^https://etcd\.io/docs/v3\.5/install/?$ - -# Unreliable from GitHub runners; these pages remain browser-accessible. -^https://cert-manager\.io/docs/?$ -^https://catalog\.ngc\.nvidia\.com/orgs/nvidia/teams/ai-dynamo/ -^https://helm\.sh/docs/intro/install/?$ -^https://book\.kubebuilder\.io/cronjob-tutorial/webhook-implementation\.html$ diff --git a/deploy/helm/charts/platform/README.md b/deploy/helm/charts/platform/README.md index 99d08e51b8c3..fef49ba1cb6d 100644 --- a/deploy/helm/charts/platform/README.md +++ b/deploy/helm/charts/platform/README.md @@ -115,8 +115,8 @@ The chart includes built-in validation to prevent all operator conflicts: | global.grove.enabled | bool | `false` | Whether to enable Grove integration (multinode orchestration via PodCliqueSets). Set to true when Grove is available in the cluster (installed externally). Automatically true when install=true. The operator uses this to decide whether to create PodCliqueSets for multinode deployments. | | dynamo-operator.enabled | bool | `true` | Whether to enable the Dynamo Kubernetes operator deployment | | dynamo-operator.upgradeCRD | bool | `true` | Whether to manage CRDs via a pre-install/pre-upgrade hook Job. The Job runs the operator image with the crd-apply tool to apply CRDs via server-side apply. | -| dynamo-operator.natsAddr | string | `""` | NATS server address for operator communication (leave empty to use the bundled NATS chart). Format: "nats://hostname:port" | -| dynamo-operator.etcdAddr | string | `""` | etcd server address for an external etcd instance. Only needed when using external etcd without the bundled subchart. Format: "http://hostname:port" or "https://hostname:port" | +| dynamo-operator.natsAddr | string | `""` | NATS server address for operator communication (leave empty to use the bundled NATS chart). Format: "nats://hostname:4222" | +| dynamo-operator.etcdAddr | string | `""` | etcd server address for an external etcd instance. Only needed when using external etcd without the bundled subchart. Format: "http://hostname:2379" or "https://hostname:2379" | | dynamo-operator.modelExpressURL | string | `""` | URL for the Model Express server if not deployed by this helm chart. This is ignored if Model Express server is installed by this helm chart (global.model-express.enabled is true). | | dynamo-operator.namespaceRestriction | object | `{"enabled":false,"lease":{"duration":"30s","renewInterval":"10s"},"targetNamespace":null}` | DEPRECATED: Namespace-restricted mode is deprecated and will be removed in a future release. Use cluster-wide mode (the default) instead. Do not enable this for new deployments. | | dynamo-operator.namespaceRestriction.enabled | bool | `false` | DEPRECATED: Do not enable for new deployments. Namespace-restricted mode is deprecated. | diff --git a/deploy/helm/charts/platform/values.yaml b/deploy/helm/charts/platform/values.yaml index 8993f049c009..a5153b826ebc 100644 --- a/deploy/helm/charts/platform/values.yaml +++ b/deploy/helm/charts/platform/values.yaml @@ -64,10 +64,10 @@ dynamo-operator: # Environment variables to pass to operator Deployment. env: [] - # -- NATS server address for operator communication (leave empty to use the bundled NATS chart). Format: "nats://hostname:port" + # -- NATS server address for operator communication (leave empty to use the bundled NATS chart). Format: "nats://hostname:4222" natsAddr: "" - # -- etcd server address for an external etcd instance. Only needed when using external etcd without the bundled subchart. Format: "http://hostname:port" or "https://hostname:port" + # -- etcd server address for an external etcd instance. Only needed when using external etcd without the bundled subchart. Format: "http://hostname:2379" or "https://hostname:2379" etcdAddr: "" # -- URL for the Model Express server if not deployed by this helm chart. This is ignored if Model Express server is installed by this helm chart (global.model-express.enabled is true). From ba61ce88cbb243553bc4ac3b2a2a39206117555d Mon Sep 17 00:00:00 2001 From: Ishan Dhanani Date: Sun, 5 Jul 2026 12:11:20 +0000 Subject: [PATCH 25/30] revert: remove Lychee CI workaround Signed-off-by: Ishan Dhanani --- .github/workflows/docs-link-check.yml | 1 - deploy/helm/charts/platform/README.md | 4 ++-- deploy/helm/charts/platform/values.yaml | 4 ++-- 3 files changed, 4 insertions(+), 5 deletions(-) diff --git a/.github/workflows/docs-link-check.yml b/.github/workflows/docs-link-check.yml index 9ce0af918f74..5cb0a21370f8 100644 --- a/.github/workflows/docs-link-check.yml +++ b/.github/workflows/docs-link-check.yml @@ -45,7 +45,6 @@ jobs: --host-concurrency 10 --host-request-interval 1s --host-stats - ${{ github.event_name == 'pull_request' && '--offline' || '' }} . fail: true env: diff --git a/deploy/helm/charts/platform/README.md b/deploy/helm/charts/platform/README.md index fef49ba1cb6d..99d08e51b8c3 100644 --- a/deploy/helm/charts/platform/README.md +++ b/deploy/helm/charts/platform/README.md @@ -115,8 +115,8 @@ The chart includes built-in validation to prevent all operator conflicts: | global.grove.enabled | bool | `false` | Whether to enable Grove integration (multinode orchestration via PodCliqueSets). Set to true when Grove is available in the cluster (installed externally). Automatically true when install=true. The operator uses this to decide whether to create PodCliqueSets for multinode deployments. | | dynamo-operator.enabled | bool | `true` | Whether to enable the Dynamo Kubernetes operator deployment | | dynamo-operator.upgradeCRD | bool | `true` | Whether to manage CRDs via a pre-install/pre-upgrade hook Job. The Job runs the operator image with the crd-apply tool to apply CRDs via server-side apply. | -| dynamo-operator.natsAddr | string | `""` | NATS server address for operator communication (leave empty to use the bundled NATS chart). Format: "nats://hostname:4222" | -| dynamo-operator.etcdAddr | string | `""` | etcd server address for an external etcd instance. Only needed when using external etcd without the bundled subchart. Format: "http://hostname:2379" or "https://hostname:2379" | +| dynamo-operator.natsAddr | string | `""` | NATS server address for operator communication (leave empty to use the bundled NATS chart). Format: "nats://hostname:port" | +| dynamo-operator.etcdAddr | string | `""` | etcd server address for an external etcd instance. Only needed when using external etcd without the bundled subchart. Format: "http://hostname:port" or "https://hostname:port" | | dynamo-operator.modelExpressURL | string | `""` | URL for the Model Express server if not deployed by this helm chart. This is ignored if Model Express server is installed by this helm chart (global.model-express.enabled is true). | | dynamo-operator.namespaceRestriction | object | `{"enabled":false,"lease":{"duration":"30s","renewInterval":"10s"},"targetNamespace":null}` | DEPRECATED: Namespace-restricted mode is deprecated and will be removed in a future release. Use cluster-wide mode (the default) instead. Do not enable this for new deployments. | | dynamo-operator.namespaceRestriction.enabled | bool | `false` | DEPRECATED: Do not enable for new deployments. Namespace-restricted mode is deprecated. | diff --git a/deploy/helm/charts/platform/values.yaml b/deploy/helm/charts/platform/values.yaml index a5153b826ebc..8993f049c009 100644 --- a/deploy/helm/charts/platform/values.yaml +++ b/deploy/helm/charts/platform/values.yaml @@ -64,10 +64,10 @@ dynamo-operator: # Environment variables to pass to operator Deployment. env: [] - # -- NATS server address for operator communication (leave empty to use the bundled NATS chart). Format: "nats://hostname:4222" + # -- NATS server address for operator communication (leave empty to use the bundled NATS chart). Format: "nats://hostname:port" natsAddr: "" - # -- etcd server address for an external etcd instance. Only needed when using external etcd without the bundled subchart. Format: "http://hostname:2379" or "https://hostname:2379" + # -- etcd server address for an external etcd instance. Only needed when using external etcd without the bundled subchart. Format: "http://hostname:port" or "https://hostname:port" etcdAddr: "" # -- URL for the Model Express server if not deployed by this helm chart. This is ignored if Model Express server is installed by this helm chart (global.model-express.enabled is true). From 925a50eb0c40ea568c23cd9e899acf5a3b875e91 Mon Sep 17 00:00:00 2001 From: Ishan Dhanani Date: Sun, 5 Jul 2026 12:11:20 +0000 Subject: [PATCH 26/30] docs(agents): use Pi session identity by default Signed-off-by: Ishan Dhanani --- docs/agents/agent-harnesses.md | 1 - 1 file changed, 1 deletion(-) diff --git a/docs/agents/agent-harnesses.md b/docs/agents/agent-harnesses.md index 5c241ee43cd9..333248efeada 100644 --- a/docs/agents/agent-harnesses.md +++ b/docs/agents/agent-harnesses.md @@ -51,7 +51,6 @@ Point it at the Dynamo OpenAI-compatible endpoint and run Pi with the `dynamo` p ```bash export DYNAMO_BASE_URL=http://localhost:8000/v1 export DYNAMO_API_KEY=dynamo-local -export DYN_AGENT_SESSION_ID=pi-demo-$(uuidgen) pi --model dynamo/zai-org/GLM-4.7-Flash ``` From 6ba54a6c990fc2350cb21a3af7cf1f682b954516 Mon Sep 17 00:00:00 2001 From: Ishan Dhanani Date: Sun, 5 Jul 2026 12:13:19 +0000 Subject: [PATCH 27/30] docs(thunderagent): clarify router model name Signed-off-by: Ishan Dhanani --- .../src/dynamo/thunderagent_router/run_minimax_8xh100.sh | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/components/src/dynamo/thunderagent_router/run_minimax_8xh100.sh b/components/src/dynamo/thunderagent_router/run_minimax_8xh100.sh index d4bf8d152ea7..9d7975743937 100755 --- a/components/src/dynamo/thunderagent_router/run_minimax_8xh100.sh +++ b/components/src/dynamo/thunderagent_router/run_minimax_8xh100.sh @@ -19,8 +19,8 @@ cleanup() { trap cleanup EXIT MODEL_PATH="${MODEL_PATH:-MiniMaxAI/MiniMax-M2.7}" -MODEL_NAME="${MODEL_NAME:-MiniMaxAI/MiniMax-M2}" -WORKER_MODEL="$MODEL_NAME" +MODEL_NAME_ROUTER="${MODEL_NAME_ROUTER:-MiniMaxAI/MiniMax-M2}" +WORKER_MODEL="$MODEL_NAME_ROUTER" BLOCK_SIZE=16 HTTP_PORT=8100 @@ -58,7 +58,7 @@ python -m dynamo.vllm \ if [[ "$POLICY" == "ta" ]]; then DYN_SYSTEM_PORT=8183 python -m dynamo.thunderagent_router \ --endpoint dynamo.backend.generate \ - --model-name "$MODEL_NAME" \ + --model-name "$MODEL_NAME_ROUTER" \ --model-path "$MODEL_PATH" \ --dyn-tool-call-parser minimax_m2 \ --dyn-reasoning-parser minimax_append_think \ @@ -82,7 +82,7 @@ until curl -fsS "http://127.0.0.1:${HTTP_PORT}/v1/models/${WORKER_MODEL}/ready" | jq -e '([.namespaces[].worker_types.aggregated.workers // 0] | add) == 2' >/dev/null; do sleep 5 done -until curl -fsS "http://127.0.0.1:${HTTP_PORT}/v1/models" 2>/dev/null | grep -Fq "$MODEL_NAME"; do +until curl -fsS "http://127.0.0.1:${HTTP_PORT}/v1/models" 2>/dev/null | grep -Fq "$MODEL_NAME_ROUTER"; do sleep 5 done echo "$POLICY stack ready at http://127.0.0.1:${HTTP_PORT}/v1" From 13a636d02191948234fe403625f4cc702ded9c10 Mon Sep 17 00:00:00 2001 From: Ishan Dhanani Date: Sun, 5 Jul 2026 12:16:18 +0000 Subject: [PATCH 28/30] fix(thunderagent): stop launcher on child exit Signed-off-by: Ishan Dhanani --- .../dynamo/thunderagent_router/run_minimax_8xh100.sh | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/components/src/dynamo/thunderagent_router/run_minimax_8xh100.sh b/components/src/dynamo/thunderagent_router/run_minimax_8xh100.sh index 9d7975743937..6e73746040bf 100755 --- a/components/src/dynamo/thunderagent_router/run_minimax_8xh100.sh +++ b/components/src/dynamo/thunderagent_router/run_minimax_8xh100.sh @@ -7,16 +7,16 @@ # ./run_minimax_8xh100.sh kv set -euo pipefail +SCRIPT_DIR="$(dirname "$(readlink -f "$0")")" +source "$SCRIPT_DIR/../../../../examples/common/launch_utils.sh" + POLICY="${1:-}" if [[ "$POLICY" != "ta" && "$POLICY" != "kv" ]]; then echo "usage: $0 ta|kv" >&2 exit 2 fi -cleanup() { - jobs -pr | xargs -r kill -} -trap cleanup EXIT +trap dynamo_exit_trap EXIT MODEL_PATH="${MODEL_PATH:-MiniMaxAI/MiniMax-M2.7}" MODEL_NAME_ROUTER="${MODEL_NAME_ROUTER:-MiniMaxAI/MiniMax-M2}" @@ -87,4 +87,4 @@ until curl -fsS "http://127.0.0.1:${HTTP_PORT}/v1/models" 2>/dev/null | grep -Fq done echo "$POLICY stack ready at http://127.0.0.1:${HTTP_PORT}/v1" -wait +wait_any_exit From 5e5dfe25300d0e3bfebad4960afa70d8a1aac8d4 Mon Sep 17 00:00:00 2001 From: Ishan Dhanani Date: Sun, 5 Jul 2026 12:21:09 +0000 Subject: [PATCH 29/30] fix(sglang): isolate HiCache capacity publication Signed-off-by: Ishan Dhanani --- components/src/dynamo/sglang/register.py | 27 +++++---- .../sglang/tests/test_runtime_metadata.py | 55 +++++++++++++++++++ 2 files changed, 72 insertions(+), 10 deletions(-) diff --git a/components/src/dynamo/sglang/register.py b/components/src/dynamo/sglang/register.py index eae6c10b9712..6ee3cb913def 100644 --- a/components/src/dynamo/sglang/register.py +++ b/components/src/dynamo/sglang/register.py @@ -464,14 +464,6 @@ async def _get_runtime_config( try: scheduler_info = engine._scheduler_init_result.scheduler_infos[0] - hicache_capacity_runtime_data = _get_hicache_capacity_runtime_data( - scheduler_info - ) - if hicache_capacity_runtime_data is not None: - runtime_config.set_engine_specific( - SGLANG_HICACHE_CAPACITY_RUNTIME_KEY, - json.dumps(hicache_capacity_runtime_data), - ) capacity = runtime_capacity(server_args, scheduler_info) max_total_tokens = scheduler_info.get("max_total_num_tokens") @@ -500,12 +492,27 @@ async def _get_runtime_config( f"{unpublished} will not be published; SGLang will use its internal defaults." ) - return runtime_config - except Exception as e: logging.warning(f"Failed to get runtime config: {e}. Proceeding without it.") return runtime_config + try: + hicache_capacity_runtime_data = _get_hicache_capacity_runtime_data( + scheduler_info + ) + if hicache_capacity_runtime_data is not None: + runtime_config.set_engine_specific( + SGLANG_HICACHE_CAPACITY_RUNTIME_KEY, + json.dumps(hicache_capacity_runtime_data), + ) + logging.info("Published SGLang HiCache capacity runtime metadata.") + except Exception as e: + logging.warning( + "Failed to attach SGLang HiCache capacity runtime metadata: %s", e + ) + + return runtime_config + async def register_model_with_readiness_gate( engine: sgl.Engine, diff --git a/components/src/dynamo/sglang/tests/test_runtime_metadata.py b/components/src/dynamo/sglang/tests/test_runtime_metadata.py index f8757de81361..19aef6bcfe01 100644 --- a/components/src/dynamo/sglang/tests/test_runtime_metadata.py +++ b/components/src/dynamo/sglang/tests/test_runtime_metadata.py @@ -87,3 +87,58 @@ def test_hicache_capacity_runtime_data_ignores_invalid_values(value): assert ( _get_hicache_capacity_runtime_data({"hicache_host_total_tokens": value}) is None ) + + +@pytest.mark.asyncio +async def test_hicache_publish_failure_preserves_core_capacity(monkeypatch, caplog): + from dynamo.sglang import register + + server_args = SimpleNamespace( + context_length=4096, + disaggregation_mode=None, + max_prefill_tokens=None, + page_size=16, + speculative_algorithm="NONE", + speculative_num_steps=None, + ) + dynamo_args = register.DynamoConfig() + dynamo_args.enable_local_indexer = False + scheduler_info = { + "hicache_host_total_tokens": 300, + "max_total_num_tokens": 1024, + } + engine = SimpleNamespace( + _scheduler_init_result=SimpleNamespace(scheduler_infos=[scheduler_info]) + ) + capacity = SimpleNamespace( + max_num_seqs=None, + max_num_batched_tokens=1024, + total_kv_blocks=64, + ) + + monkeypatch.setattr(register, "model_card_dp_rank_bounds", lambda _: (0, 1)) + monkeypatch.setattr(register, "get_sglang_worker_group_id", lambda _: None) + monkeypatch.setattr( + register, "_get_bootstrap_info_for_config", lambda _: (None, None) + ) + monkeypatch.setattr(register, "_get_mooncake_runtime_data", lambda _: None) + monkeypatch.setattr(register, "runtime_capacity", lambda *_: capacity) + + original_set = register.ModelRuntimeConfig.set_engine_specific + + def fail_hicache_publish(self, key, value): + if key == register.SGLANG_HICACHE_CAPACITY_RUNTIME_KEY: + raise RuntimeError("publish failed") + return original_set(self, key, value) + + monkeypatch.setattr( + register.ModelRuntimeConfig, "set_engine_specific", fail_hicache_publish + ) + + runtime_config = await register._get_runtime_config( + engine, server_args, dynamo_args + ) + + assert runtime_config.total_kv_blocks == 64 + assert runtime_config.max_num_batched_tokens == 1024 + assert "Failed to attach SGLang HiCache capacity runtime metadata" in caplog.text From 394be5b670ec1dcec87300e69d48acab8959499d Mon Sep 17 00:00:00 2001 From: Ishan Dhanani Date: Sun, 5 Jul 2026 18:50:45 +0000 Subject: [PATCH 30/30] docs: follow Harbor adapter layout Signed-off-by: Ishan Dhanani --- .../src/dynamo/thunderagent_router/README.md | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/components/src/dynamo/thunderagent_router/README.md b/components/src/dynamo/thunderagent_router/README.md index d141cc5a2493..769e58772af8 100644 --- a/components/src/dynamo/thunderagent_router/README.md +++ b/components/src/dynamo/thunderagent_router/README.md @@ -100,7 +100,7 @@ agent. This walkthrough runs the same SWE-bench Verified task through ThunderAgent and the stock Dynamo KV router. Harbor owns the task container, Pi runs inside it, and the model stack runs on the host. ThunderAgent is backend-agnostic and works with Dynamo's vLLM and SGLang backends; this example uses one 8-GPU node and two TP4 vLLM workers loading `MiniMaxAI/MiniMax-M2.7` from Hugging Face and serving the API alias `MiniMaxAI/MiniMax-M2`. There is no HiCache, Mooncake, shared cache, or frontend admission control in either arm. -The [agent-plugins `DynamoPi` adapter](https://github.com/ai-dynamo/agent-plugins/blob/main/pi-plugin/harbor_dynamo_pi.py) is required. It installs the Dynamo provider in each Harbor task container and maps Harbor's per-trial ID to one stable `x-dynamo-session-id` across all Pi turns. The ThunderAgent arm also sends one terminal session request so the router can release the completed program; the stock KV arm disables that request because it has no lifecycle consumer. +The [agent-plugins `DynamoPi` adapter](https://github.com/ai-dynamo/agent-plugins/blob/main/pi-plugin/harbor/dynamo_pi.py) is required. It installs the Dynamo provider in each Harbor task container and maps Harbor's per-trial ID to one stable `x-dynamo-session-id` across all Pi turns. The ThunderAgent arm also sends one terminal session request so the router can release the completed program; the stock KV arm disables that request because it has no lifecycle consumer. ### 1. Install the three source trees @@ -111,7 +111,7 @@ git clone https://github.com/ai-dynamo/dynamo ~/src/dynamo git clone --branch v0.16.0 --depth 1 https://github.com/harbor-framework/harbor ~/src/harbor git clone https://github.com/ai-dynamo/agent-plugins ~/src/agent-plugins -git -C ~/src/agent-plugins checkout cd0434136a92522532ee3ae5bf61078f47e6761a +git -C ~/src/agent-plugins checkout 223a0b8823610d042e7479bfff9b93eeac4a23ec cd ~/src/dynamo uv venv --python 3.12 --seed .venv @@ -163,13 +163,13 @@ export DYN_AGENT_SESSION_FINAL=1 # export DYN_AGENT_SESSION_FINAL=0 export PI_PLUGIN_DIR=~/src/agent-plugins/pi-plugin -export PYTHONPATH=$PI_PLUGIN_DIR +export PYTHONPATH=$PI_PLUGIN_DIR/harbor export DYNAMO_BASE_URL=http://$(ip route get 1.1.1.1 | awk '{print $7; exit}'):8100/v1 curl -fsS "$DYNAMO_BASE_URL/models" harbor run \ -d swebench-verified@1.0 -i astropy__astropy-12907 \ - -a harbor_dynamo_pi:DynamoPi -m dynamo/MiniMaxAI/MiniMax-M2 \ + -a dynamo_pi:DynamoPi -m dynamo/MiniMaxAI/MiniMax-M2 \ --ak version=0.72.1 --ae "DYNAMO_BASE_URL=$DYNAMO_BASE_URL" \ --ae "DYN_AGENT_SESSION_FINAL=$DYN_AGENT_SESSION_FINAL" \ --mounts "[{\"type\":\"bind\",\"source\":\"$PI_PLUGIN_DIR\",\"target\":\"/opt/pi-dynamo-provider\",\"read_only\":true}]" \ @@ -189,7 +189,7 @@ docker login harbor run \ -d swebench-verified@1.0 \ -a nop \ - --extra-docker-compose ~/src/agent-plugins/pi-plugin/harbor-host-network.yml \ + --extra-docker-compose ~/src/agent-plugins/pi-plugin/harbor/host-network.yml \ -n 32 --n-concurrent-agents 32 \ --memory ignore \ --no-delete --environment-kwarg keep_containers=true \ @@ -203,11 +203,11 @@ The host-network overlay avoids creating one Docker bridge network per trial, wh ```bash harbor run \ -d swebench-verified@1.0 \ - -a harbor_dynamo_pi:DynamoPi -m dynamo/MiniMaxAI/MiniMax-M2 \ + -a dynamo_pi:DynamoPi -m dynamo/MiniMaxAI/MiniMax-M2 \ --ak version=0.72.1 --ae "DYNAMO_BASE_URL=$DYNAMO_BASE_URL" \ --ae "DYN_AGENT_SESSION_FINAL=$DYN_AGENT_SESSION_FINAL" \ --mounts "[{\"type\":\"bind\",\"source\":\"$PI_PLUGIN_DIR\",\"target\":\"/opt/pi-dynamo-provider\",\"read_only\":true}]" \ - --extra-docker-compose ~/src/agent-plugins/pi-plugin/harbor-host-network.yml \ + --extra-docker-compose ~/src/agent-plugins/pi-plugin/harbor/host-network.yml \ -n 256 --n-concurrent-agents 256 \ --agent-setup-timeout-multiplier 10 --memory ignore \ --no-delete --environment-kwarg keep_containers=true \