From da36145a2531d8d99cbedc5f7e53a17b452cfee9 Mon Sep 17 00:00:00 2001 From: Kai Ma Date: Mon, 18 May 2026 11:06:06 -0400 Subject: [PATCH 1/2] feat(planner): power planner stress testbed (alpha + gamma) Adds the synthetic-metrics testbed: deterministic, no GPU, no cluster. Alpha-class (27 scenarios, A1-F26): synthetic fleet + fake metrics + fake actuator drive the planner through fault-injection scenarios covering AIC drift, NVML clamps, K8s RBAC denials, node loss / recovery, Prometheus outages, MDC gaps, budget shrinkage, AIC infeasibility, and drift-threshold boundary cases. Gamma-class (3 scenarios, G1-G3): mocker-driven trace replay with synthetic-power overlay (replay/synthetic_power_overlay.py) and a power-aware replay adapter (replay/power_aware_replay_adapter.py) exercise the closed loop against real Mooncake traces. Infrastructure: - runner.py + scenarios.py + assertions.py + recorder.py + clock.py (run-loop, scenario loader, invariant checks, recording). - synthetic_fleet.py + fake_actuator.py + fake_aic.py + fake_planner_metrics.py + fake_prometheus.py (test doubles for every external dependency of the planner run loop). - _runtime_stub.py installs a stub dynamo._core when the compiled Rust binding is absent, so the testbed runs on developer laptops without a CUDA toolchain (carries every dynamo._core symbol used by dynamo.llm at module-load time, including the post-rebase RoutingConstraints addition from main PR #9558). - grafana/testbed_dashboard.json + systems/ (h100_pcie / h100_sxm / h200_sxm SKUs) + traces/placeholder_h200_disagg_1rps.jsonl provide a complete observability + replay stack. Production code: - offline/replay_adapter.py: 9-line timing fix (now_s = max(tick.at_s, bridge_now_s)) prevents stale-tick loops on sparse traces. 86 testbed tests + 30 scenarios (27 alpha + 3 gamma) ship green at this tip (1 skipped pending env-var; test_aic_real_data.py is module-skipped unless AIC_SANDBOX_DIR is set). Part of the PR #9369 split (PR 4 of 6). See docs/design-docs/pr9369-split-plan.md. Signed-off-by: Kai Ma --- .../dynamo/planner/offline/replay_adapter.py | 9 +- .../dynamo/planner/tests/testbed/README.md | 329 +++++++++ .../dynamo/planner/tests/testbed/__init__.py | 28 + .../planner/tests/testbed/_runtime_stub.py | 252 +++++++ .../planner/tests/testbed/assertions.py | 323 +++++++++ .../src/dynamo/planner/tests/testbed/clock.py | 27 + .../dynamo/planner/tests/testbed/conftest.py | 89 +++ .../planner/tests/testbed/fake_actuator.py | 184 +++++ .../dynamo/planner/tests/testbed/fake_aic.py | 113 ++++ .../tests/testbed/fake_planner_metrics.py | 182 +++++ .../planner/tests/testbed/fake_prometheus.py | 126 ++++ .../testbed/grafana/testbed_dashboard.json | 414 ++++++++++++ .../dynamo/planner/tests/testbed/recorder.py | 194 ++++++ .../planner/tests/testbed/replay/__init__.py | 3 + .../replay/power_aware_replay_adapter.py | 518 ++++++++++++++ .../testbed/replay/replay_fake_actuator.py | 150 +++++ .../testbed/replay/synthetic_power_overlay.py | 217 ++++++ .../dynamo/planner/tests/testbed/runner.py | 633 ++++++++++++++++++ .../dynamo/planner/tests/testbed/scenarios.py | 604 +++++++++++++++++ .../A1_power_under_estimate_decode.yaml | 25 + .../A2_power_over_estimate_prefill.yaml | 28 + .../scenarios/A3_ttft_under_estimate.yaml | 29 + .../scenarios/A4_step_drift_midstream.yaml | 31 + .../A5_oscillating_drift_sub_interval.yaml | 27 + .../A6_coefficient_pegged_at_clamp.yaml | 24 + .../B10_daemonset_absent_one_node.yaml | 29 + .../B11_frontend_post_partial_failure.yaml | 32 + .../testbed/scenarios/B7_nvml_clamp_low.yaml | 29 + .../testbed/scenarios/B8_nvml_clamp_high.yaml | 29 + .../testbed/scenarios/B9_k8s_rbac_denied.yaml | 20 + .../scenarios/C12_one_node_down_low_load.yaml | 29 + .../C13_one_node_down_high_load.yaml | 33 + .../C14_all_decode_workers_fail.yaml | 35 + .../testbed/scenarios/C15_node_recovery.yaml | 34 + .../scenarios/C16_warmup_power_spike.yaml | 40 ++ .../scenarios/D17_prometheus_outage.yaml | 26 + .../scenarios/D18_prometheus_stale.yaml | 30 + .../scenarios/D19_dcgm_attribution_loss.yaml | 26 + .../D20_mdc_missing_max_batched_tokens.yaml | 22 + ...21_prom_window_cross_after_cap_change.yaml | 42 ++ .../scenarios/E21_budget_shrunk_live.yaml | 24 + .../E22_budget_below_min_endpoint.yaml | 19 + .../E23_aic_infeasible_at_startup.yaml | 21 + .../E24_aic_exception_at_runtime.yaml | 25 + .../E25_aic_5_consecutive_failures.yaml | 25 + .../F26_drift_threshold_boundary.yaml | 55 ++ .../scenarios/G1_realistic_decode_drift.yaml | 32 + .../G2_scheduler_driven_scale_out.yaml | 30 + .../G3_scheduler_power_cap_interaction.yaml | 30 + .../scenarios/_base/h100_pcie_disagg.yaml | 47 ++ .../testbed/scenarios/_base/h200_agg.yaml | 47 ++ .../testbed/scenarios/_base/h200_disagg.yaml | 53 ++ .../scenarios/_base/mocker_h200_disagg.yaml | 63 ++ .../_base/mocker_h200_synthetic_workload.yaml | 54 ++ .../planner/tests/testbed/synthetic_fleet.py | 410 ++++++++++++ .../tests/testbed/systems/h100_pcie.yaml | 18 + .../tests/testbed/systems/h100_sxm.yaml | 18 + .../tests/testbed/systems/h200_sxm.yaml | 21 + .../planner/tests/testbed/test_scenarios.py | 110 +++ .../planner/tests/testbed/tests/__init__.py | 0 .../tests/testbed/tests/test_aic_real_data.py | 508 ++++++++++++++ .../planner/tests/testbed/tests/test_fakes.py | 284 ++++++++ .../tests/testbed/tests/test_overlay.py | 180 +++++ .../testbed/tests/test_scenarios_loadable.py | 82 +++ .../testbed/tests/test_self_consistency.py | 162 +++++ .../planner/tests/testbed/traces/README.md | 82 +++ .../traces/placeholder_h200_disagg_1rps.jsonl | 300 +++++++++ 67 files changed, 7714 insertions(+), 1 deletion(-) create mode 100644 components/src/dynamo/planner/tests/testbed/README.md create mode 100644 components/src/dynamo/planner/tests/testbed/__init__.py create mode 100644 components/src/dynamo/planner/tests/testbed/_runtime_stub.py create mode 100644 components/src/dynamo/planner/tests/testbed/assertions.py create mode 100644 components/src/dynamo/planner/tests/testbed/clock.py create mode 100644 components/src/dynamo/planner/tests/testbed/conftest.py create mode 100644 components/src/dynamo/planner/tests/testbed/fake_actuator.py create mode 100644 components/src/dynamo/planner/tests/testbed/fake_aic.py create mode 100644 components/src/dynamo/planner/tests/testbed/fake_planner_metrics.py create mode 100644 components/src/dynamo/planner/tests/testbed/fake_prometheus.py create mode 100644 components/src/dynamo/planner/tests/testbed/grafana/testbed_dashboard.json create mode 100644 components/src/dynamo/planner/tests/testbed/recorder.py create mode 100644 components/src/dynamo/planner/tests/testbed/replay/__init__.py create mode 100644 components/src/dynamo/planner/tests/testbed/replay/power_aware_replay_adapter.py create mode 100644 components/src/dynamo/planner/tests/testbed/replay/replay_fake_actuator.py create mode 100644 components/src/dynamo/planner/tests/testbed/replay/synthetic_power_overlay.py create mode 100644 components/src/dynamo/planner/tests/testbed/runner.py create mode 100644 components/src/dynamo/planner/tests/testbed/scenarios.py create mode 100644 components/src/dynamo/planner/tests/testbed/scenarios/A1_power_under_estimate_decode.yaml create mode 100644 components/src/dynamo/planner/tests/testbed/scenarios/A2_power_over_estimate_prefill.yaml create mode 100644 components/src/dynamo/planner/tests/testbed/scenarios/A3_ttft_under_estimate.yaml create mode 100644 components/src/dynamo/planner/tests/testbed/scenarios/A4_step_drift_midstream.yaml create mode 100644 components/src/dynamo/planner/tests/testbed/scenarios/A5_oscillating_drift_sub_interval.yaml create mode 100644 components/src/dynamo/planner/tests/testbed/scenarios/A6_coefficient_pegged_at_clamp.yaml create mode 100644 components/src/dynamo/planner/tests/testbed/scenarios/B10_daemonset_absent_one_node.yaml create mode 100644 components/src/dynamo/planner/tests/testbed/scenarios/B11_frontend_post_partial_failure.yaml create mode 100644 components/src/dynamo/planner/tests/testbed/scenarios/B7_nvml_clamp_low.yaml create mode 100644 components/src/dynamo/planner/tests/testbed/scenarios/B8_nvml_clamp_high.yaml create mode 100644 components/src/dynamo/planner/tests/testbed/scenarios/B9_k8s_rbac_denied.yaml create mode 100644 components/src/dynamo/planner/tests/testbed/scenarios/C12_one_node_down_low_load.yaml create mode 100644 components/src/dynamo/planner/tests/testbed/scenarios/C13_one_node_down_high_load.yaml create mode 100644 components/src/dynamo/planner/tests/testbed/scenarios/C14_all_decode_workers_fail.yaml create mode 100644 components/src/dynamo/planner/tests/testbed/scenarios/C15_node_recovery.yaml create mode 100644 components/src/dynamo/planner/tests/testbed/scenarios/C16_warmup_power_spike.yaml create mode 100644 components/src/dynamo/planner/tests/testbed/scenarios/D17_prometheus_outage.yaml create mode 100644 components/src/dynamo/planner/tests/testbed/scenarios/D18_prometheus_stale.yaml create mode 100644 components/src/dynamo/planner/tests/testbed/scenarios/D19_dcgm_attribution_loss.yaml create mode 100644 components/src/dynamo/planner/tests/testbed/scenarios/D20_mdc_missing_max_batched_tokens.yaml create mode 100644 components/src/dynamo/planner/tests/testbed/scenarios/D21_prom_window_cross_after_cap_change.yaml create mode 100644 components/src/dynamo/planner/tests/testbed/scenarios/E21_budget_shrunk_live.yaml create mode 100644 components/src/dynamo/planner/tests/testbed/scenarios/E22_budget_below_min_endpoint.yaml create mode 100644 components/src/dynamo/planner/tests/testbed/scenarios/E23_aic_infeasible_at_startup.yaml create mode 100644 components/src/dynamo/planner/tests/testbed/scenarios/E24_aic_exception_at_runtime.yaml create mode 100644 components/src/dynamo/planner/tests/testbed/scenarios/E25_aic_5_consecutive_failures.yaml create mode 100644 components/src/dynamo/planner/tests/testbed/scenarios/F26_drift_threshold_boundary.yaml create mode 100644 components/src/dynamo/planner/tests/testbed/scenarios/G1_realistic_decode_drift.yaml create mode 100644 components/src/dynamo/planner/tests/testbed/scenarios/G2_scheduler_driven_scale_out.yaml create mode 100644 components/src/dynamo/planner/tests/testbed/scenarios/G3_scheduler_power_cap_interaction.yaml create mode 100644 components/src/dynamo/planner/tests/testbed/scenarios/_base/h100_pcie_disagg.yaml create mode 100644 components/src/dynamo/planner/tests/testbed/scenarios/_base/h200_agg.yaml create mode 100644 components/src/dynamo/planner/tests/testbed/scenarios/_base/h200_disagg.yaml create mode 100644 components/src/dynamo/planner/tests/testbed/scenarios/_base/mocker_h200_disagg.yaml create mode 100644 components/src/dynamo/planner/tests/testbed/scenarios/_base/mocker_h200_synthetic_workload.yaml create mode 100644 components/src/dynamo/planner/tests/testbed/synthetic_fleet.py create mode 100644 components/src/dynamo/planner/tests/testbed/systems/h100_pcie.yaml create mode 100644 components/src/dynamo/planner/tests/testbed/systems/h100_sxm.yaml create mode 100644 components/src/dynamo/planner/tests/testbed/systems/h200_sxm.yaml create mode 100644 components/src/dynamo/planner/tests/testbed/test_scenarios.py create mode 100644 components/src/dynamo/planner/tests/testbed/tests/__init__.py create mode 100644 components/src/dynamo/planner/tests/testbed/tests/test_aic_real_data.py create mode 100644 components/src/dynamo/planner/tests/testbed/tests/test_fakes.py create mode 100644 components/src/dynamo/planner/tests/testbed/tests/test_overlay.py create mode 100644 components/src/dynamo/planner/tests/testbed/tests/test_scenarios_loadable.py create mode 100644 components/src/dynamo/planner/tests/testbed/tests/test_self_consistency.py create mode 100644 components/src/dynamo/planner/tests/testbed/traces/README.md create mode 100644 components/src/dynamo/planner/tests/testbed/traces/placeholder_h200_disagg_1rps.jsonl diff --git a/components/src/dynamo/planner/offline/replay_adapter.py b/components/src/dynamo/planner/offline/replay_adapter.py index 7da6d0385ce6..a741e738f0e7 100644 --- a/components/src/dynamo/planner/offline/replay_adapter.py +++ b/components/src/dynamo/planner/offline/replay_adapter.py @@ -363,7 +363,14 @@ def _build_tick_input( self, tick: ScheduledTick, result: dict[str, Any] ) -> TickInput: """Convert bridge result dict to planner TickInput.""" - now_s = result["now_ms"] / 1000.0 + # Use the scheduled tick time as the lower bound for now_s. The Rust + # bridge returns now_ms = last-event-completion time, which can be well + # below tick.at_s when the trace is sparse (e.g., the placeholder trace + # has one event every 12 s but throughput ticks fire every 5 s). If + # we used now_ms directly, the state machine would keep scheduling the + # next throughput tick at the same sub-second offset forever. + bridge_now_s = result["now_ms"] / 1000.0 + now_s = max(tick.at_s, bridge_now_s) worker_counts = None if tick.need_worker_states: diff --git a/components/src/dynamo/planner/tests/testbed/README.md b/components/src/dynamo/planner/tests/testbed/README.md new file mode 100644 index 000000000000..582279175e99 --- /dev/null +++ b/components/src/dynamo/planner/tests/testbed/README.md @@ -0,0 +1,329 @@ +# Power Planner Stress Testbed + +Pure in-memory stress testbed for the Dynamo Power Planner. No GPUs, no +Kubernetes, no Prometheus server required — runs on any developer laptop in +< 30 s (α-class) or < 2 min (γ-class). + +See the design document for full rationale: +`docs/design-docs/powerplanner-testbed-design.md` + +--- + +## Quick start + +```bash +# Install test dependencies (from repo root) +pip install -e "components/[testbed]" + +# Run all α-class scenarios (fast, ~30 s) +pytest components/src/dynamo/planner/tests/testbed/ -v -m "testbed and not gamma" + +# Run the full suite including γ-class (requires dynamo.llm / mocker) +pytest components/src/dynamo/planner/tests/testbed/ -v -m testbed + +# Run a single scenario by name +pytest components/src/dynamo/planner/tests/testbed/test_scenarios.py::test_alpha[A1_power_under_estimate_decode] -v + +# CLI: run one scenario, dump CSV + plot +python -m dynamo.planner.tests.testbed.runner \ + --scenario A1 \ + --csv out/A1.csv \ + --plot out/A1.png \ + --prom-textfile /tmp/A1.prom + +# CLI: run the whole catalog +python -m dynamo.planner.tests.testbed.runner --all --csv-dir out/ +``` + +--- + +## Architecture + +``` +testbed/ +├── __init__.py # import guard (blocks pynvml) +├── conftest.py # blocks real K8s writes (session-scoped) +├── clock.py # deterministic virtual clock +├── scenarios.py # Pydantic models + YAML loader (extends: support) +├── runner.py # ScenarioRunner: dispatches α / γ +├── recorder.py # TickSnapshot + TickHistory (CSV / Prom / plot) +├── assertions.py # assertion DSL evaluator +│ +├── synthetic_fleet.py # α: truth model (power + latency + capacity) +├── fake_prometheus.py # SHARED: FakePrometheusClient +├── fake_actuator.py # α: FakeActuator (PlannerConnector impl) +├── fake_aic.py # SHARED: FakeAIC + per-system Pareto tables +├── fake_planner_metrics.py # SHARED: in-memory Counter/Gauge mocks +│ +├── replay/ # γ-class extension (mocker-based) +│ ├── synthetic_power_overlay.py +│ ├── replay_fake_actuator.py +│ └── power_aware_replay_adapter.py +│ +├── traces/ # γ trace fixtures +│ ├── placeholder_h200_disagg_1rps.jsonl +│ └── README.md +│ +├── systems/ # per-SKU hardware constants +│ ├── h200_sxm.yaml +│ ├── h100_sxm.yaml +│ └── h100_pcie.yaml +│ +├── scenarios/ # 27-scenario catalog +│ ├── _base/ # inheritance templates +│ └── A1_*.yaml … G3_*.yaml +│ +├── test_scenarios.py # pytest parametrize over scenarios/*.yaml +└── tests/ # tests OF the testbed itself + ├── test_fakes.py + ├── test_overlay.py + ├── test_scenarios_loadable.py + └── test_self_consistency.py +``` + +### Test classes + +| Class | Driver | Observability | Scheduler | Speed | +|-------|--------|---------------|-----------|-------| +| **α** | `SyntheticFleet` (in-memory truth model) | `FakePrometheusClient` | Deterministic (no Kubernetes) | < 1 s / scenario | +| **γ** | `dynamo-mocker` replay + `SyntheticPowerOverlay` | Same FakePrometheusClient (power from overlay) | Real mocker scheduler | 2–4 s / scenario | +| **real-AIC** | Real `aiconfigurator` perf database from a mounted sandbox | `MagicMock` metrics; in-process EMA loop | None (drives `update_correction()` directly) | < 20 s / suite | + +#### real-AIC class + +Opt-in only. Exercises `AICPowerOptimizer.optimize()` and the EMA drift loop +against a real AIC perf database with measured `power_w` data — the closest +thing to a production loop we can run without a real GPU. See §8 row 14 of +`powerplanner-design.md` for the defensive clamp this class regression-tests. + +```bash +# Mount or symlink your AIC power-data tree at .aic_sandbox/systems/ first +# (h200_sxm.yaml + data/h200_sxm/...; same layout as aiconfigurator/systems/). +AIC_SANDBOX_DIR="$PWD/.aic_sandbox/systems" \ + pytest components/src/dynamo/planner/tests/testbed/tests/test_aic_real_data.py -v +``` + +What it verifies (parametrized across every SKU present in the sandbox): + +* `AIConfiguratorPerfEstimator.estimate_perf` returns non-zero `power_w` + for both prefill and decode (i.e. the sandbox actually has power data, + not a TDP-fallback fixture). +* `AICPowerOptimizer.optimize()` produces a `PowerAwareConfig` whose + per-GPU caps never exceed `TDP × _COEFF_MAX = 2 × TDP`. +* `aic_power_w_clamped_total{side=...}` fires iff AIC's raw `power_w` + exceeded `TDP × 1.1` for that side. On the H200 vLLM 0.19.1 data this + is exercised; on B200 TRT-LLM 1.3.0rc6 the data is dense enough that + the clamp stays cold. +* The EMA loop converges in three regimes — well-calibrated → 1.0, + over-predict → pegs at 0.5, under-predict → 1.5 — against H200's + *real* AIC denominators. +* `should_reoptimize()` respects the hysteresis count under sustained + SLA breach and stays silent under healthy load. + +CI runs this class by setting `AIC_SANDBOX_DIR` in the job environment; +local invocations without the env var are silently skipped at module +import time with a one-line reason. The marker is `@pytest.mark.real_aic`. + +γ-class skips automatically in two situations: + +1. **No `dynamo._core` native binding** (e.g., fresh dev box without `maturin`): + the runtime stub installs a no-op `dynamo._core` and `conftest.py`'s + `pytest_collection_modifyitems` hook skips every `@pytest.mark.gamma` test. + See [Appendix C.10](../../../../../../docs/design-docs/powerplanner-testbed-design.md). +2. **Older bridge API only** (`create_disagg` without `from_synthetic_disagg`): + the α–γ cross-validation test (`test_alpha_gamma_agree_on_decode_drift`) + skips because the placeholder trace fallback can't drive AIC drift. The + three γ scenarios + `test_gamma_no_bias` still run. See + [Appendix D.7](../../../../../../docs/design-docs/powerplanner-testbed-design.md). + +Expected outcomes by environment: + +| Environment | Pass | Skip | +|-------------|-----:|-----:| +| Local box (no Rust mocker) | 82 | 5 (all γ) | +| Dev pod (older `create_disagg` bridge) | 86 | 1 (α–γ cross-validation only) | +| Future CI box (newer `from_synthetic_disagg`) | 87 | 0 | + +--- + +## Scenario catalog + +| Group | Scenarios | What's stress-tested | +|-------|-----------|---------------------| +| A | A1–A6 | AIC drift / mis-calibration | +| B | B7–B11 | Actuation failures (NVML clamp, K8s RBAC, DaemonSet absent, frontend POST) | +| C | C12–C16 | Node / pod failures and recovery | +| D | D17–D21 | Observability / data-plane failures (Prometheus outage, stale, DCGM loss, MDC, window-cross) | +| E | E21–E25 | Budget / config edge cases | +| F | F26 | Drift threshold boundary | +| G | G1–G3 | γ-class: realistic scheduler-driven scenarios | + +Total: 27 scenarios (24 α + 3 γ). + +--- + +## Adding a new α scenario (< 1 hour) + +1. Copy a base template: + ```bash + cp scenarios/_base/h200_disagg.yaml scenarios/X27_my_scenario.yaml + ``` + +2. Edit the YAML — set `name:`, `description:`, `events:`, and `assertions:`. + +3. Check the assertion field names are valid: + ```bash + pytest tests/testbed/tests/test_scenarios_loadable.py -v + ``` + +4. Run your new scenario: + ```bash + pytest "test_scenarios.py::test_alpha[X27_my_scenario]" -v + ``` + +### Available event types + +Each event is a dict in `events:` with a `type:` discriminator. Single source +of truth: `scenarios.py` (the Pydantic `Event` union). + +| `type` | Required fields | Effect | +|--------|----------------|--------| +| `bias_step` | `at_tick`, `signal`, `value` | One-shot bias change on `signal` (`power_bias_decode`, `power_bias_prefill`, `ttft_bias`, `itl_bias`, `capacity_bias`). Optional `auto_inject_window_cross: true` schedules a one-tick Prom-aggregation-window-cross at `at_tick+1`. | +| `bias_ramp` | `start_tick`, `end_tick`, `signal`, `from`, `to` | Linear ramp on `signal` over `[start_tick, end_tick]`. | +| `bias_sine` | `signal`, `amplitude`, `period_ticks`, `offset` (optional) | Sustained sinusoidal perturbation. | +| `actuation_fault` | `at_tick`, `duration_ticks`, `mode` | `mode ∈ {rbac_denied, nvml_low, nvml_high, daemonset_absent}`. | +| `node_down` | `at_tick`, `n_prefill_lost`, `n_decode_lost` | Yank replicas without going through the planner. | +| `node_up` | `at_tick`, `n_prefill_restored`, `n_decode_restored` | Restore lost replicas. | +| `prom_outage` | `at_tick`, `duration_ticks`, `signals:` (list) | Force `get_*` to return `None` for the listed Prometheus signals. | +| `prom_stale` | `at_tick`, `duration_ticks`, `lag_ticks` | Read returns the value from `tick − lag_ticks`. | +| `prom_window_cross_event` | `at_tick`, `signal`, `weight_old` | Models a Prom aggregation window that spans a cap or replica change. | +| `budget_change` | `at_tick`, `new_total_w` | Shrink / expand `total_gpu_power_limit` live. | +| `frontend_post_fault` | `at_tick`, `duration_ticks`, `failing_fraction` | Probabilistically fail `/busy_threshold` POSTs. | +| `mdc_unavailable` | `at_tick`, `duration_ticks` | MDC returns no `max_batched_tokens` (admission goes implied-only). | +| `aic_failure` | `at_tick`, `mode`, `n_consecutive` | `mode ∈ {empty_pareto, raises}`; resets after `n_consecutive` ticks. | + +### Available assertion predicates + +Every assertion needs **exactly one** of `at_tick:` / `always: true` / +`eventually_by_tick:`. The Pydantic validator rejects ambiguous or empty +predicates at load time (no more silent skips). + +```yaml +# Point-in-time +- at_tick: 50 + field: c_power_d + op: ">=" + value: 1.2 + +# Always (every tick) +- always: true # MUST be `true`; bare `always:` is rejected + field: projected_w + op: "<=" + ref: "planner.total_gpu_power_limit" + +# Eventually +- eventually_by_tick: 80 + field: sweep_fired + op: "==" + value: 1.0 + +# Expression (restricted AST eval; allowed roots: history, planner, counters, abs, min, max, len) +- expr: "history[-1].c_power_d > history[10].c_power_d" + always: true + description: "coefficient increases over time" +``` + +Valid `op` values: `==`, `!=`, `<`, `<=`, `>`, `>=`, `within`. +For `within`, also set `tolerance:` (relative tolerance). + +Valid `ref:` prefixes: `planner.`, `counters.`, `fleet.`, `overlay.`. + +--- + +## Adding a new γ scenario (< 3 hours) + +1. Obtain or generate a mocker trace and place it in `traces/`: + ```bash + dynamo-mocker \ + --config examples/deployments/powerplanner/h200_disagg.yaml \ + --duration 120s \ + --dump-trace \ + components/src/dynamo/planner/tests/testbed/traces/my_trace.jsonl + ``` + +2. Copy a γ base template: + ```bash + cp scenarios/_base/mocker_h200_disagg.yaml scenarios/G4_my_gamma_scenario.yaml + ``` + +3. Set `mocker.trace_file: components/src/dynamo/planner/tests/testbed/traces/my_trace.jsonl` + and add your `overlay.bias`, `events:`, and `assertions:`. + + The trace must be in **Mooncake format** (`timestamp` / `input_length` / + `output_length` / `hash_ids` per line) — see `traces/README.md`. + +4. Validate and run: + ```bash + pytest tests/testbed/tests/test_scenarios_loadable.py -v + pytest "test_scenarios.py::test_gamma[G4_my_gamma_scenario]" -v + ``` + +--- + +## Grafana dashboard + +A pre-built Grafana dashboard JSON is in: +`grafana/testbed_dashboard.json` + +It expects Prometheus textfile metrics written by: +```bash +python -m dynamo.planner.tests.testbed.runner --all \ + --prom-textfile /var/lib/node_exporter/textfile_collector/testbed.prom +``` + +Import into Grafana: **Dashboards → Import → Upload JSON file**. + +Panels: +- `c_power_d` / `c_power_p` correction coefficients over ticks +- `cap_d` / `cap_p` applied power caps +- `n_d` / `n_p` replica counts +- `projected_w` vs `budget_w` (power budget utilization) +- `sweep_fired` events +- Per-scenario status heatmap + +Filter by `scenario` label to view a single scenario or compare across them. + +--- + +## Mutation gate (code review checklist) + +If you rename a field in `recorder.py::TickSnapshot`, the `test_scenarios_loadable.py` +gate will fail at pytest collection and list every affected scenario — fix the +YAML references before merging. + +If you change `aic_power_optimizer.py`'s EMA α coefficient, scenario A4 +(`A4_step_drift_midstream.yaml`) should fail on its half-life assertion. +Run it explicitly before merging AIC tuning changes: +```bash +pytest "test_scenarios.py::test_alpha[A4_step_drift_midstream]" -v -s +``` + +--- + +## Hardware safety + +The testbed has two layers of protection: + +1. **Import guard** (`__init__.py`): installs a `sys.meta_path` finder that + raises `ImportError` if anything tries to `import pynvml` _after_ the + testbed package has been imported. Because the parent `dynamo.planner` + package is usually imported first, this is best-effort and primarily + documentary — production code never reaches `pynvml` anyway. + +2. **K8s write block** (`conftest.py`): a session-scoped autouse fixture + monkeypatches `kubernetes.client.CoreV1Api.patch_namespaced_pod` and + `create_namespaced_pod` to raise `RuntimeError` for the entire pytest + session. This is the actual safety net. + +Neither layer affects production code; both are active only when running under +`pytest` inside this package. diff --git a/components/src/dynamo/planner/tests/testbed/__init__.py b/components/src/dynamo/planner/tests/testbed/__init__.py new file mode 100644 index 000000000000..8e09d625278b --- /dev/null +++ b/components/src/dynamo/planner/tests/testbed/__init__.py @@ -0,0 +1,28 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Power Planner stress testbed — pure in-memory simulation. + +NEVER imports pynvml. NEVER instantiates kubernetes.client.CoreV1Api in a way +that can issue real requests. Asserted at import time + pytest session. + +See docs/design-docs/powerplanner-testbed-design.md for the full design. +""" + +import sys + +_BANNED = ("pynvml",) + + +class _ImportGuard: + def find_spec(self, name, path, target=None): + if name in _BANNED: + raise ImportError( + f"Testbed forbids import of {name!r}: this would risk " + f"touching real hardware. See docs/design-docs/" + f"powerplanner-testbed-design.md §1.3." + ) + return None + + +sys.meta_path.insert(0, _ImportGuard()) diff --git a/components/src/dynamo/planner/tests/testbed/_runtime_stub.py b/components/src/dynamo/planner/tests/testbed/_runtime_stub.py new file mode 100644 index 000000000000..c3b01e073585 --- /dev/null +++ b/components/src/dynamo/planner/tests/testbed/_runtime_stub.py @@ -0,0 +1,252 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Install a stub for ``dynamo._core`` when the native binding is unavailable. + +The α-class testbed deliberately avoids the Rust runtime (no NATS, no etcd, +no Rust scheduler). The only reason it would fail to import without this +shim is that ``dynamo.planner/__init__.py`` eagerly pulls in Kubernetes +connectors → ``dynamo.runtime.logging`` → ``dynamo._core.log_message``. + +This module installs a minimal in-memory stand-in for ``dynamo._core`` so +the planner package imports cleanly on a developer laptop without the +maturin-built extension. If the real ``dynamo._core`` is available (CI, +production), the stub is **not** installed. + +Hardware-safety remains intact: nothing in the stub can talk to a real +NATS/etcd cluster or NVML; calls into any stubbed symbol log and return +sentinel values. + +Idempotent — safe to import / call multiple times. +""" +from __future__ import annotations + +import sys +import types + + +def _ensure_dynamo_runtime_on_path() -> None: + """Add ``lib/bindings/python/src`` to ``sys.path`` so the pure-Python + ``dynamo.runtime`` package is importable. + + ``dynamo.runtime`` is not installed as a distribution package on dev + boxes — it lives under ``lib/bindings/python/src`` in the repo. Its + only native dependency is ``dynamo._core.log_message``, which is already + stubbed to a no-op, so the module imports cleanly once the path is + present. + + Idempotent — skips if ``dynamo.runtime`` is already importable or if the + path is already in ``sys.path``. + """ + if "dynamo.runtime" in sys.modules: + return + # Walk up from this file's location to the repo root, then locate the + # pure-Python bindings. Path: /components/src/dynamo/planner/tests/testbed/ + # → up 6 levels → / → lib/bindings/python/src + import pathlib + + this_file = pathlib.Path(__file__).resolve() + # Ascend: testbed → tests → planner → dynamo → src → components → + repo_root = this_file.parents[6] + bindings_src = repo_root / "lib" / "bindings" / "python" / "src" + if bindings_src.is_dir(): + bindings_str = str(bindings_src) + if bindings_str not in sys.path: + sys.path.insert(0, bindings_str) + + +def install_stub_if_needed() -> bool: + """Install the stub iff the real ``dynamo._core`` cannot be imported. + + Returns True if a stub was installed (or already in place); False if the + real binding is loaded. + """ + if "dynamo._core" in sys.modules: + # Some other code path beat us to it. Honor whatever is there. + return getattr(sys.modules["dynamo._core"], "_TESTBED_STUB", False) + + try: + import dynamo._core # noqa: F401 — real binding present + + return False + except ImportError: + pass + + # ------------------------------------------------------------------ + # Build the stub module. + # ------------------------------------------------------------------ + stub = types.ModuleType("dynamo._core") + stub._TESTBED_STUB = True + stub.__doc__ = ( + "Testbed stand-in for dynamo._core. Installed by " + "dynamo.planner.tests.testbed._runtime_stub when the native binding " + "is unavailable. Any call to a stubbed symbol raises immediately — " + "the testbed must never reach the runtime layer." + ) + + # --- Functions --- + def log_message(*args, **kwargs): + # Rust's env_logger sink; the Python LogHandler forwards records here. + # Silently drop — tests don't care about runtime log output. + return None + + stub.log_message = log_message + + def _unimplemented(name): + def _raise(*_args, **_kwargs): + raise NotImplementedError( + f"dynamo._core.{name} is stubbed for the testbed — " + f"the α-class harness must not reach the runtime layer. " + f"If you need this in a γ-class scenario, build the real " + f"binding (maturin develop in lib/bindings/python)." + ) + + return _raise + + # --- Classes consumed at module-import time by dynamo.runtime --- + # These are *imported* but never *instantiated* during testbed runs; + # making them stub classes satisfies the import-time binding check. + for _name in ( + "Client", + "Context", + "DistributedRuntime", + "Endpoint", + # dynamo.planner.connectors.virtual imports this at top level + "VirtualConnectorCoordinator", + # dynamo.llm imports (only needed for γ; harmless to stub) + "AicPerfConfig", + "EngineType", + "EntrypointArgs", + "FpmDirectPublisher", + "FpmEventRelay", + "FpmEventSubscriber", + "HttpAsyncEngine", + "HttpService", + "KserveGrpcService", + "KvEventPublisher", + "KvRouter", + "KvRouterConfig", + "LoRADownloader", + "MediaDecoder", + "MediaFetcher", + "MockEngineArgs", + "ModelCardInstanceId", + "ModelInput", + "ModelRuntimeConfig", + "ModelType", + "OverlapScores", + "PlannerReplayBridge", + "PythonAsyncEngine", + "RadixTree", + "ReasoningConfig", + "RouterConfig", + "RouterMode", + # Added on main (PR #9558) — referenced at dynamo.llm import time. + "RoutingConstraints", + "SglangArgs", + "WorkerMetricsPublisher", + "ModelDeploymentCard", + # Exceptions used in dynamo.llm.exceptions + "Cancelled", + "CannotConnect", + "ConnectionTimeout", + "Disconnected", + "DynamoException", + "EngineShutdown", + "InvalidArgument", + "StreamIncomplete", + "Unknown", + ): + setattr(stub, _name, type(_name, (), {"__init__": _unimplemented(_name)})) + + # --- Free functions used at module-import time --- + for _name in ( + "compute_block_hash_for_seq", + "fetch_model", + "lora_name_to_id", + "make_engine", + "register_model", + "run_input", + "run_kv_indexer", + "run_mocker_trace_replay", + "unregister_model", + ): + setattr(stub, _name, _unimplemented(_name)) + + sys.modules["dynamo._core"] = stub + + # With dynamo._core now stubbed, the pure-Python dynamo.runtime package + # can import cleanly — but only if its source directory is on sys.path. + # Add it now (idempotent; no-ops if real binding already installed it). + _ensure_dynamo_runtime_on_path() + + return True + + +def install_predictor_deps_stub_if_needed() -> None: + """Stub heavy ML predictor deps (``pmdarima``/``filterpy``/``prophet``). + + ``dynamo.planner.core.load.predictors`` imports them at module load, + even though the testbed only ever uses the ``constant`` predictor. + These deps are large and not part of a standard developer install, + so we provide do-nothing stubs to unblock import. + + Idempotent. If the real package is available, nothing happens. + """ + if "pmdarima" not in sys.modules: + try: + import pmdarima # noqa: F401 + except ImportError: + pmd = types.ModuleType("pmdarima") + pmd._TESTBED_STUB = True + + def _no_predictor(*_a, **_kw): + raise NotImplementedError( + "pmdarima is stubbed for the testbed (constant predictor only)." + ) + + pmd.auto_arima = _no_predictor + pmd.ARIMA = type("ARIMA", (), {"__init__": _no_predictor}) + sys.modules["pmdarima"] = pmd + + if "filterpy" not in sys.modules: + try: + import filterpy.kalman # noqa: F401 + except ImportError: + fp = types.ModuleType("filterpy") + fp.__path__ = [] # mark as package + fp_kal = types.ModuleType("filterpy.kalman") + fp_kal._TESTBED_STUB = True + fp_kal.KalmanFilter = type( + "KalmanFilter", + (), + { + "__init__": lambda *a, **kw: (_ for _ in ()).throw( + NotImplementedError( + "filterpy is stubbed for the testbed (constant predictor only)." + ) + ) + }, + ) + sys.modules["filterpy"] = fp + sys.modules["filterpy.kalman"] = fp_kal + fp.kalman = fp_kal + + if "prophet" not in sys.modules: + try: + import prophet # noqa: F401 + except ImportError: + pr = types.ModuleType("prophet") + pr._TESTBED_STUB = True + pr.Prophet = type( + "Prophet", + (), + { + "__init__": lambda *a, **kw: (_ for _ in ()).throw( + NotImplementedError( + "prophet is stubbed for the testbed (constant predictor only)." + ) + ) + }, + ) + sys.modules["prophet"] = pr diff --git a/components/src/dynamo/planner/tests/testbed/assertions.py b/components/src/dynamo/planner/tests/testbed/assertions.py new file mode 100644 index 000000000000..165c9e803d44 --- /dev/null +++ b/components/src/dynamo/planner/tests/testbed/assertions.py @@ -0,0 +1,323 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Assertion evaluation engine for testbed scenarios. + +Supports the structured form (field / op / value / tolerance / ref) and the +expression fallback form (expr with restricted AST eval). + +Structured predicates: + ``at_tick: N`` — evaluated at exactly tick N + ``always:`` — must hold at every tick + ``eventually_by_tick: N`` — must hold at some tick ≤ N + +Expression predicates use ``ast.literal_eval``-restricted eval with: + ``history`` — list of TickSnapshot objects + ``planner`` — PlannerSpec dict + ``counters`` — dict of cumulative counter deltas + +Assertion failure raises AssertionError with a descriptive message. +""" + +from __future__ import annotations + +import ast +import math +from typing import TYPE_CHECKING, Any, Optional + +if TYPE_CHECKING: + from dynamo.planner.tests.testbed.recorder import TickHistory, TickSnapshot + from dynamo.planner.tests.testbed.scenarios import ( + ExprAssertion, + ScenarioSpec, + StructuredAssertion, + ) + + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + +_ALLOWED_NAMES = frozenset( + {"history", "planner", "counters", "abs", "min", "max", "len"} +) +_ALLOWED_NODE_TYPES = ( + ast.Expression, + ast.BoolOp, + ast.UnaryOp, + ast.BinOp, + ast.Compare, + ast.Call, + ast.Attribute, + ast.Subscript, + ast.Index, + ast.Name, + ast.Constant, # ast.Num/ast.Str deprecated in 3.8, removed in 3.14 + ast.And, + ast.Or, + ast.Not, + ast.Gt, + ast.GtE, + ast.Lt, + ast.LtE, + ast.Eq, + ast.NotEq, + ast.Add, + ast.Sub, + ast.Mult, + ast.Div, + ast.Mod, + ast.USub, + ast.UAdd, + ast.Load, +) + + +def _safe_eval(expr: str, context: dict[str, Any]) -> Any: + """Evaluate expression in restricted AST context.""" + tree = ast.parse(expr, mode="eval") + for node in ast.walk(tree): + if not isinstance(node, _ALLOWED_NODE_TYPES): + raise ValueError( + f"Disallowed AST node type {type(node).__name__} in expr: {expr!r}" + ) + if ( + isinstance(node, ast.Name) + and node.id not in _ALLOWED_NAMES + and node.id not in context + ): + raise ValueError(f"Unknown name {node.id!r} in expr: {expr!r}") + return eval(compile(tree, "", "eval"), {"__builtins__": {}}, context) + + +def _get_field(snap: "TickSnapshot", field: str) -> Any: + return getattr(snap, field) + + +def _get_ref(ref: str, scenario: "ScenarioSpec") -> float: + parts = ref.split(".") + if parts[0] == "planner": + return float(getattr(scenario.planner, parts[1])) + raise ValueError(f"Unknown ref prefix: {ref!r}") + + +def _apply_op(actual: float, op: str, expected: float, tolerance: float = 0.0) -> bool: + if op == "<": + return actual < expected + elif op == "<=": + return actual <= expected + elif op == "==": + return math.isclose(actual, expected, rel_tol=1e-6) + elif op == ">=": + return actual >= expected + elif op == ">": + return actual > expected + elif op == "!=": + return not math.isclose(actual, expected, rel_tol=1e-6) + elif op == "within": + return abs(actual - expected) <= tolerance * max(1.0, abs(expected)) + return False + + +# --------------------------------------------------------------------------- +# Main evaluator +# --------------------------------------------------------------------------- + + +def evaluate_all( + history: "TickHistory", + scenario: "ScenarioSpec", + counters: Optional[dict[str, Any]] = None, +) -> list[str]: + """Evaluate all scenario assertions against the recorded history. + + Returns a list of failure messages (empty list = all passed). + + ``counters`` is optional and exposed as ``counters`` inside ``expr:`` + assertions; α-runner currently doesn't track external counter deltas so + the default empty dict is fine. + """ + failures: list[str] = [] + parsed = scenario.parsed_assertions() + counters = counters or {} + + for assertion in parsed: + from dynamo.planner.tests.testbed.scenarios import ( + ExprAssertion, + StructuredAssertion, + ) + + if isinstance(assertion, StructuredAssertion): + _eval_structured(assertion, history, scenario, counters, failures) + elif isinstance(assertion, ExprAssertion): + _eval_expr(assertion, history, scenario, counters, failures) + + return failures + + +# Back-compat alias for older imports; remove once no callers remain. +evaluate_assertions = evaluate_all + + +def _eval_structured( + a: "StructuredAssertion", + history: "TickHistory", + scenario: "ScenarioSpec", + counters: dict[str, Any], + failures: list[str], +) -> None: + desc = a.description or f"field={a.field} op={a.op} value={a.value}" + + # Resolve expected value + if a.value is not None: + expected = a.value + elif a.ref is not None: + try: + expected = _get_ref(a.ref, scenario) + except Exception as e: + failures.append(f"[{desc}] Could not resolve ref {a.ref!r}: {e}") + return + else: + failures.append(f"[{desc}] Assertion has neither value nor ref") + return + + tolerance = a.tolerance or 0.0 + + # Select ticks to evaluate + if a.at_tick is not None: + tick_idx = a.at_tick + if tick_idx >= len(history): + failures.append( + f"[{desc}] at_tick={tick_idx} but history only has {len(history)} ticks" + ) + return + snaps = [history[tick_idx]] + _check_snaps( + snaps, a.field, a.op, expected, tolerance, desc, failures, mode="at_tick" + ) + + elif a.always is True: + snaps = list(history.snapshots) + _check_snaps( + snaps, a.field, a.op, expected, tolerance, desc, failures, mode="always" + ) + + elif a.eventually_by_tick is not None: + limit = min(a.eventually_by_tick, len(history) - 1) + snaps = history.snapshots[: limit + 1] + _check_snaps_eventually( + snaps, a.field, a.op, expected, tolerance, desc, failures + ) + + +def _check_snaps( + snaps: list["TickSnapshot"], + field: str, + op: str, + expected: float, + tolerance: float, + desc: str, + failures: list[str], + mode: str, +) -> None: + for snap in snaps: + actual = _get_field(snap, field) + if not _apply_op(float(actual), op, expected, tolerance): + failures.append( + f"[{desc}] FAIL at tick {snap.tick}: {field}={actual!r} {op} {expected}" + + (f" ±{tolerance * 100:.0f}%" if op == "within" else "") + + f" ({mode})" + ) + if mode == "at_tick": + return + + +def _check_snaps_eventually( + snaps: list["TickSnapshot"], + field: str, + op: str, + expected: float, + tolerance: float, + desc: str, + failures: list[str], +) -> None: + for snap in snaps: + actual = _get_field(snap, field) + if _apply_op(float(actual), op, expected, tolerance): + return + if snaps: + last = snaps[-1] + failures.append( + f"[{desc}] FAIL: {field} never satisfied {op} {expected} by tick {last.tick}" + ) + else: + failures.append(f"[{desc}] FAIL: no snapshots to check") + + +def _eval_expr( + a: "ExprAssertion", + history: "TickHistory", + scenario: "ScenarioSpec", + counters: dict[str, Any], + failures: list[str], +) -> None: + desc = a.description or f"expr={a.expr!r}" + context: dict[str, Any] = { + "history": history.snapshots, + "planner": scenario.planner.model_dump(), + "counters": counters, + "abs": abs, + "min": min, + "max": max, + "len": len, + } + + if a.at_tick is not None: + tick_idx = a.at_tick + if tick_idx >= len(history): + failures.append( + f"[{desc}] at_tick={tick_idx} but history only has {len(history)} ticks" + ) + return + _eval_expr_at(a.expr, context, desc, failures, tick_idx) + + elif a.always is True: + for snap in history.snapshots: + context["history"] = history.snapshots + if not _eval_expr_at( + a.expr, context, desc, failures, snap.tick, silent=True + ): + failures.append(f"[{desc}] FAIL at tick {snap.tick}") + + elif a.eventually_by_tick is not None: + for snap in history.snapshots[: a.eventually_by_tick + 1]: + try: + if _safe_eval(a.expr, context): + return + except Exception: + pass + failures.append( + f"[{desc}] FAIL: expression never true by tick {a.eventually_by_tick}" + ) + + +def _eval_expr_at( + expr: str, + context: dict[str, Any], + desc: str, + failures: list[str], + tick: int, + silent: bool = False, +) -> bool: + try: + result = _safe_eval(expr, context) + if not result: + if not silent: + failures.append( + f"[{desc}] FAIL at tick {tick}: expr evaluated to {result!r}" + ) + return False + return True + except Exception as e: + failures.append(f"[{desc}] ERROR evaluating expr at tick {tick}: {e}") + return False diff --git a/components/src/dynamo/planner/tests/testbed/clock.py b/components/src/dynamo/planner/tests/testbed/clock.py new file mode 100644 index 000000000000..1e37c7d6b93b --- /dev/null +++ b/components/src/dynamo/planner/tests/testbed/clock.py @@ -0,0 +1,27 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Deterministic clock for the testbed. + +Replaces ``time.monotonic()`` inside ``AICPowerOptimizer`` so that rate-limit +semantics for ``aic_reoptimize_interval`` are fully controllable. The current +tick × interval_s gives a stable monotonic value without relying on wall-clock. +""" + +from __future__ import annotations + + +class Clock: + """Deterministic virtual clock tied to the scenario tick counter.""" + + def __init__(self, interval_s: float = 60.0) -> None: + self._interval_s = interval_s + self._tick: int = 0 + + def advance(self, tick: int) -> None: + """Called by the runner at the start of each tick.""" + self._tick = tick + + def now(self) -> float: + """Return virtual monotonic time in seconds.""" + return self._tick * self._interval_s diff --git a/components/src/dynamo/planner/tests/testbed/conftest.py b/components/src/dynamo/planner/tests/testbed/conftest.py new file mode 100644 index 000000000000..66fdf22b773c --- /dev/null +++ b/components/src/dynamo/planner/tests/testbed/conftest.py @@ -0,0 +1,89 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Testbed pytest configuration. + +Three responsibilities, in strict order: + +1. Install a stub for ``dynamo._core`` if the maturin-built native binding + isn't available on this machine. This MUST happen before any test module + imports ``dynamo.planner.*`` (which transitively pulls in the runtime). +2. Block real K8s writes via session-scoped autouse fixture. +3. Register testbed-specific markers so ``--strict-markers`` is happy. +""" + +import importlib.util +import pathlib +import sys + +# --------------------------------------------------------------------------- +# Step 1 — install the dynamo._core stub at conftest collection time. +# +# We load the stub module via importlib *bypassing* the dynamo.planner +# package, otherwise we'd trigger the exact import chain we're trying to +# avoid (planner.__init__ → connectors.kubernetes_api → dynamo.runtime +# → dynamo._core). +# --------------------------------------------------------------------------- +_STUB_PATH = pathlib.Path(__file__).parent / "_runtime_stub.py" +_spec = importlib.util.spec_from_file_location("_runtime_stub_isolated", _STUB_PATH) +_stub_mod = importlib.util.module_from_spec(_spec) +sys.modules["_runtime_stub_isolated"] = _stub_mod +_spec.loader.exec_module(_stub_mod) +_stub_installed = _stub_mod.install_stub_if_needed() + +from unittest.mock import patch # noqa: E402 + +import pytest # noqa: E402 + + +def pytest_configure(config: pytest.Config) -> None: + """Register testbed-only markers.""" + config.addinivalue_line( + "markers", "testbed: synthetic-metrics power planner stress testbed" + ) + config.addinivalue_line( + "markers", + "gamma: γ-class scenarios (need mocker / dynamo.llm with native binding)", + ) + config.addinivalue_line( + "markers", + "real_aic: exercises AICPowerOptimizer against a real AIC perf database " + "(opt-in; set AIC_SANDBOX_DIR env var to a populated systems/ directory " + "and install the aiconfigurator package)", + ) + + +def pytest_collection_modifyitems( + config: pytest.Config, items: list[pytest.Item] +) -> None: + """Skip γ-class tests when only the ``dynamo._core`` stub is installed. + + γ-class scenarios depend on the real Rust mocker (`PlannerReplayBridge`), + which only exists when the maturin-built native binding is on + ``sys.path``. ``importorskip("dynamo.llm")`` doesn't work here because + ``dynamo.llm`` is pure-Python and importable even with the stub: we have + to look at the stub flag explicitly. + """ + if not _stub_installed: + return + skip_marker = pytest.mark.skip( + reason="γ-class test requires real dynamo._core (mocker / " + "PlannerReplayBridge); only the testbed stub is installed on " + "this machine." + ) + for item in items: + if "gamma" in item.keywords: + item.add_marker(skip_marker) + + +@pytest.fixture(autouse=True, scope="session") +def _block_real_k8s_writes(): + """Monkeypatch K8s pod-write paths to raise — defense in depth.""" + with patch( + "kubernetes.client.CoreV1Api.patch_namespaced_pod", + side_effect=RuntimeError("Testbed: real K8s writes are forbidden"), + ), patch( + "kubernetes.client.CoreV1Api.create_namespaced_pod", + side_effect=RuntimeError("Testbed: real K8s writes are forbidden"), + ): + yield diff --git a/components/src/dynamo/planner/tests/testbed/fake_actuator.py b/components/src/dynamo/planner/tests/testbed/fake_actuator.py new file mode 100644 index 000000000000..07b6cd25e76a --- /dev/null +++ b/components/src/dynamo/planner/tests/testbed/fake_actuator.py @@ -0,0 +1,184 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""FakeActuator — α-class PlannerConnector implementation. + +Replaces KubernetesConnector + Power Agent NVML for the testbed. All +actuation lands in in-memory state. Fault hooks inject: + - ``rbac_denied`` → set_component_replicas raises 403-style RuntimeError + - ``nvml_low`` → cap clamped up to sku_min_w + - ``nvml_high`` → cap clamped down to sku_max_w + - ``daemonset_absent``→ patch_pod_annotation silently no-ops (annotation recorded, + no side-effect in truth model → truth draws at TDP) + - ``frontend_post`` → post_busy_threshold raises for a fraction of calls + +The γ-class subclass (ReplayFakeActuator) is in replay/replay_fake_actuator.py. +""" + +from __future__ import annotations + +import random +from dataclasses import dataclass +from typing import TYPE_CHECKING, Optional + +from dynamo.planner.config.defaults import SubComponentType +from dynamo.planner.connectors.base import PlannerConnector + +if TYPE_CHECKING: + from dynamo.planner.tests.testbed.fake_planner_metrics import FakePlannerMetrics + from dynamo.planner.tests.testbed.scenarios import ScenarioSpec, SystemSpec + from dynamo.planner.tests.testbed.synthetic_fleet import SyntheticFleet + + +@dataclass +class AppliedCaps: + cap_p: int + cap_d: int + + +class FakeActuator(PlannerConnector): + """α-class actuator — pure in-memory state. + + Tracks applied replica counts and per-GPU caps. Reflects applied caps + back into the SyntheticFleet truth model so the closed loop is consistent. + """ + + def __init__( + self, + scenario: "ScenarioSpec", + fleet: "SyntheticFleet", + metrics: "FakePlannerMetrics", + system_spec: "SystemSpec", + ) -> None: + self._scenario = scenario + self._fleet = fleet + self._metrics = metrics + self._sku_min_w = int(system_spec.sku_min_w) + self._sku_max_w = int(system_spec.sku_max_w) + + self._applied_n_p: int = 1 + self._applied_n_d: int = ( + getattr(scenario.fleet, "gpus_per_decode_engine", 4) + if scenario.fleet + else 4 + ) + self._applied_cap_p: int = scenario.planner.prefill_engine_gpu_power_limit + self._applied_cap_d: int = scenario.planner.decode_engine_gpu_power_limit + self._annotations: dict[str, dict[str, str]] = {} # pod_name → {key: value} + + # Fault state (controlled by runner via events) + self._actuation_fault_mode: Optional[str] = None + + # ------------------------------------------------------------------ + # PlannerConnector ABC + # ------------------------------------------------------------------ + + async def add_component( + self, sub_component_type: SubComponentType, blocking: bool = True + ) -> None: + if self._actuation_fault_mode == "rbac_denied": + raise RuntimeError("403 Forbidden (synthetic actuation fault)") + if sub_component_type.name.lower() == "prefill": + self._applied_n_p += 1 + self._fleet.state.n_p_truth += 1 + else: + self._applied_n_d += 1 + self._fleet.state.n_d_truth += 1 + + async def remove_component( + self, sub_component_type: SubComponentType, blocking: bool = True + ) -> None: + if self._actuation_fault_mode == "rbac_denied": + raise RuntimeError("403 Forbidden (synthetic actuation fault)") + if sub_component_type.name.lower() == "prefill": + self._applied_n_p = max(0, self._applied_n_p - 1) + self._fleet.state.n_p_truth = max(0, self._fleet.state.n_p_truth - 1) + else: + self._applied_n_d = max(0, self._applied_n_d - 1) + self._fleet.state.n_d_truth = max(0, self._fleet.state.n_d_truth - 1) + + # ------------------------------------------------------------------ + # Testbed-specific methods + # ------------------------------------------------------------------ + + def apply_replicas(self, n_p: int, n_d: int) -> None: + """Apply desired replica counts (post power-budget clamp).""" + if self._actuation_fault_mode == "rbac_denied": + raise RuntimeError("403 Forbidden (synthetic actuation fault)") + self._applied_n_p = max(0, n_p) + self._applied_n_d = max(0, n_d) + self._fleet.state.n_p_truth = self._applied_n_p + self._fleet.state.n_d_truth = self._applied_n_d + + def apply_caps(self, cap_p: int, cap_d: int) -> None: + """Apply per-GPU power caps, injecting NVML clamp faults if active.""" + clamped_p = self._clamp(cap_p, "prefill") + clamped_d = self._clamp(cap_d, "decode") + self._applied_cap_p = clamped_p + self._applied_cap_d = clamped_d + # Reflect back to truth model + self._fleet.state.applied_cap_p = clamped_p + self._fleet.state.applied_cap_d = clamped_d + + def patch_pod_annotation(self, pod_name: str, key: str, value: str) -> None: + """Record annotation; if daemonset_absent, don't reflect to truth model.""" + if self._actuation_fault_mode == "rbac_denied": + raise RuntimeError("403 Forbidden (synthetic actuation fault)") + if pod_name not in self._annotations: + self._annotations[pod_name] = {} + self._annotations[pod_name][key] = value + if self._actuation_fault_mode != "daemonset_absent": + # Reflect the cap into truth model + try: + cap_w = int(value) + clamped = self._clamp_raw(cap_w) + # Determine which component from annotation key + if "prefill" in key.lower(): + self._fleet.state.applied_cap_p = clamped + else: + self._fleet.state.applied_cap_d = clamped + except (ValueError, TypeError): + pass + + async def post_busy_threshold( + self, pod: str, model: str, port: int, **thresholds: float + ) -> None: + """Simulate frontend POST; raise for failing_fraction of calls.""" + fault_frac = self._fleet.frontend_fault() + if fault_frac is not None: + if random.random() < fault_frac: + self._metrics.admission_partial_success_total.inc() + raise RuntimeError( + f"503 Service Unavailable (synthetic POST fault to {pod})" + ) + + def applied_caps_snapshot(self) -> AppliedCaps: + return AppliedCaps(cap_p=self._applied_cap_p, cap_d=self._applied_cap_d) + + # ------------------------------------------------------------------ + # Fault state helpers + # ------------------------------------------------------------------ + + def set_actuation_fault(self, mode: Optional[str]) -> None: + """Set the active actuation fault mode ('rbac_denied', 'nvml_low', etc.).""" + self._actuation_fault_mode = mode + + # ------------------------------------------------------------------ + # Internal + # ------------------------------------------------------------------ + + def _clamp(self, cap_w: int, component: str) -> int: + clamped = self._clamp_raw(cap_w) + if clamped > cap_w: + self._metrics.power_agent_cap_clamped_total.labels(direction="min").inc() + elif clamped < cap_w: + self._metrics.power_agent_cap_clamped_total.labels(direction="max").inc() + return clamped + + def _clamp_raw(self, cap_w: int) -> int: + mode = self._actuation_fault_mode + if mode == "nvml_low": + return max(cap_w, self._sku_min_w) + elif mode == "nvml_high": + return min(cap_w, self._sku_max_w) + return max(self._sku_min_w, min(self._sku_max_w, cap_w)) diff --git a/components/src/dynamo/planner/tests/testbed/fake_aic.py b/components/src/dynamo/planner/tests/testbed/fake_aic.py new file mode 100644 index 000000000000..3f48e82e5b0a --- /dev/null +++ b/components/src/dynamo/planner/tests/testbed/fake_aic.py @@ -0,0 +1,113 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""FakeAIC — testbed replacement for the AIConfigurator estimator. + +Injected into ``AICPowerOptimizer`` via ``optimizer._aic_estimator_factory``. +The factory ignores ``hf_id / system / backend`` and returns a +``_FakeAICEstimator`` whose responses come from the per-SKU system spec. + +Fault injection: + - ``fault_mode="normal"`` → returns system-spec constants + - ``fault_mode="raises"`` → estimate_prefill_perf raises RuntimeError + - ``fault_mode="empty_pareto"``→ returns huge TTFT (> any SLA) to trigger + infeasibility path in AICPowerOptimizer +""" + +from __future__ import annotations + +from typing import TYPE_CHECKING, Any, Callable + +if TYPE_CHECKING: + from dynamo.planner.tests.testbed.scenarios import SystemSpec + + +class _FakeDatabase: + """Minimal stand-in for AIConfiguratorPerfEstimator.database.""" + + def __init__(self, tdp_w: float) -> None: + self.system_spec = {"gpu": {"power": tdp_w}} + + +class _FakeAICEstimator: + """Minimal stand-in for AIConfiguratorPerfEstimator. + + Implements only the methods that AICPowerOptimizer.optimize() calls. + """ + + def __init__(self, system_spec: "SystemSpec", fault_mode: str) -> None: + self._spec = system_spec + self._fault_mode = fault_mode + self.database = _FakeDatabase(system_spec.tdp_w) + + def estimate_prefill_perf(self, isl: int, **kwargs: Any) -> dict[str, Any]: + if self._fault_mode == "raises": + raise RuntimeError("synthetic AIC failure (estimate_prefill_perf)") + if self._fault_mode == "empty_pareto": + # Return huge TTFT to force SLA infeasibility in the optimizer. + return {"context_latency": 999_999.0, "power_w": 0.0} + return { + "context_latency": self._spec.aic_ttft_ms, + "power_w": self._spec.aic_power_w_prefill, + } + + def get_max_kv_tokens(self, isl: int, osl: int, **kwargs: Any) -> int: + if self._fault_mode == "raises": + raise RuntimeError("synthetic AIC failure (get_max_kv_tokens)") + return self._spec.max_kv_tokens + + def estimate_perf( + self, + isl: int, + osl: int, + batch_size: int, + mode: str = "decode", + **kwargs: Any, + ) -> dict[str, Any]: + if self._fault_mode == "raises": + raise RuntimeError("synthetic AIC failure (estimate_perf)") + if self._fault_mode == "empty_pareto": + return {"tpot": 999_999.0, "power_w": 0.0} + return { + "tpot": self._spec.aic_itl_ms, + "power_w": self._spec.aic_power_w_decode, + } + + +class FakeAIC: + """Testbed AIC replacement. + + Shared between α and γ-class scenarios. The fault mode is controlled by + the scenario's active event state; the runner calls ``set_fault_mode()`` + when processing ``aic_failure`` events. + """ + + def __init__(self, system_spec: "SystemSpec") -> None: + self._system_spec = system_spec + self._fault_mode: str = "normal" + + def set_fault_mode(self, mode: str) -> None: + """Set fault mode: 'normal' | 'raises' | 'empty_pareto'.""" + assert mode in ( + "normal", + "raises", + "empty_pareto", + ), f"Unknown fault mode: {mode}" + self._fault_mode = mode + + def reset_fault(self) -> None: + self._fault_mode = "normal" + + def make_estimator_factory(self) -> Callable[..., _FakeAICEstimator]: + """Return a callable compatible with optimizer._aic_estimator_factory. + + The factory is called as ``factory(hf_id=..., system=..., backend=...)`` + and should return an estimator-like object. We capture ``self`` so the + estimator always reads the current fault mode. + """ + fake_aic = self + + def _factory(hf_id: str, system: str, backend: str) -> _FakeAICEstimator: + return _FakeAICEstimator(fake_aic._system_spec, fake_aic._fault_mode) + + return _factory diff --git a/components/src/dynamo/planner/tests/testbed/fake_planner_metrics.py b/components/src/dynamo/planner/tests/testbed/fake_planner_metrics.py new file mode 100644 index 000000000000..b350b9166369 --- /dev/null +++ b/components/src/dynamo/planner/tests/testbed/fake_planner_metrics.py @@ -0,0 +1,182 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""In-memory mock for PlannerPrometheusMetrics. + +All Prometheus Counter/Gauge/Enum operations are forwarded to simple +in-memory accumulators. The testbed reads these to assert on counter +increments and gauge values without starting a real Prometheus server. +""" + +from __future__ import annotations + +from typing import Any + + +class _FakeCounter: + """Thread-safe (enough for sync testbed) counter mock.""" + + def __init__(self) -> None: + self._labels: dict[tuple, "_FakeLabeledCounter"] = {} + self._value: float = 0.0 + + def labels(self, **kwargs: Any) -> "_FakeLabeledCounter": + key = tuple(sorted(kwargs.items())) + if key not in self._labels: + self._labels[key] = _FakeLabeledCounter() + return self._labels[key] + + def inc(self, amount: float = 1.0) -> None: + self._value += amount + + @property + def value(self) -> float: + return self._value + + def labeled_value(self, **kwargs: Any) -> float: + key = tuple(sorted(kwargs.items())) + return self._labels.get(key, _FakeLabeledCounter()).value + + +class _FakeLabeledCounter: + def __init__(self) -> None: + self._value: float = 0.0 + + def inc(self, amount: float = 1.0) -> None: + self._value += amount + + @property + def value(self) -> float: + return self._value + + +class _FakeGauge: + """Gauge mock with optional labels.""" + + def __init__(self) -> None: + self._labels: dict[tuple, "_FakeLabeledGauge"] = {} + self._value: float = 0.0 + + def labels(self, **kwargs: Any) -> "_FakeLabeledGauge": + key = tuple(sorted(kwargs.items())) + if key not in self._labels: + self._labels[key] = _FakeLabeledGauge() + return self._labels[key] + + def set(self, value: float) -> None: + self._value = value + + @property + def value(self) -> float: + return self._value + + def labeled_value(self, **kwargs: Any) -> float: + key = tuple(sorted(kwargs.items())) + return self._labels.get(key, _FakeLabeledGauge()).value + + +class _FakeLabeledGauge: + def __init__(self) -> None: + self._value: float = 0.0 + + def set(self, value: float) -> None: + self._value = value + + @property + def value(self) -> float: + return self._value + + +class _FakeEnum: + """Enum-state mock.""" + + def __init__(self) -> None: + self._state: str = "unset" + + def state(self, state: str) -> None: + self._state = state + + @property + def value(self) -> str: + return self._state + + +class FakePlannerMetrics: + """Drop-in for PlannerPrometheusMetrics. + + Provides the same attribute names; all values are backed by simple + in-memory accumulators. Read ``metrics.aic_c_ttft.value`` etc. in + assertions. + """ + + def __init__(self) -> None: + # Worker counts + self.num_prefill_replicas = _FakeGauge() + self.num_decode_replicas = _FakeGauge() + # Observed metrics + self.observed_ttft_ms = _FakeGauge() + self.observed_itl_ms = _FakeGauge() + self.observed_requests_per_second = _FakeGauge() + self.observed_request_duration_seconds = _FakeGauge() + self.observed_input_sequence_tokens = _FakeGauge() + self.observed_output_sequence_tokens = _FakeGauge() + # Predicted metrics + self.predicted_requests_per_second = _FakeGauge() + self.predicted_input_sequence_tokens = _FakeGauge() + self.predicted_output_sequence_tokens = _FakeGauge() + self.predicted_num_prefill_replicas = _FakeGauge() + self.predicted_num_decode_replicas = _FakeGauge() + # GPU usage + self.gpu_hours = _FakeGauge() + # Diagnostics latency + self.estimated_ttft_ms = _FakeGauge() + self.estimated_itl_ms = _FakeGauge() + # Engine capacity + self.engine_prefill_capacity_requests_per_second = _FakeGauge() + self.engine_decode_capacity_requests_per_second = _FakeGauge() + # Scaling decision enums + self.load_scaling_decision = _FakeEnum() + self.throughput_scaling_decision = _FakeEnum() + # FPM queue depths (labeled) + self.engine_queued_prefill_tokens = _FakeGauge() + self.engine_queued_decode_kv_tokens = _FakeGauge() + self.engine_inflight_decode_kv_tokens = _FakeGauge() + # Power-aware scaling + self.power_budget_total_watts = _FakeGauge() + self.power_projected_watts = _FakeGauge() + self.power_budget_utilization = _FakeGauge() + # AIC optimizer + self.aic_c_ttft = _FakeGauge() + self.aic_c_itl = _FakeGauge() + self.aic_c_power = _FakeGauge() + self.aic_correction_pegged_total = _FakeCounter() + self.aic_consecutive_failures = _FakeGauge() + self.aic_optimizer_exceptions_total = _FakeCounter() + self.aic_optimizer_disabled_reason = _FakeGauge() + self.aic_throughput_regression_total = _FakeCounter() + # Admission control + self.admission_implied_theta_decode = _FakeGauge() + self.admission_implied_theta_prefill_frac = _FakeGauge() + self.admission_set_theta_decode = _FakeGauge() + self.admission_set_theta_prefill_frac = _FakeGauge() + self.admission_set_theta_prefill_abs = _FakeGauge() + self.admission_max_batched_tokens_unavailable_total = _FakeCounter() + self.admission_partial_success_total = _FakeCounter() + # Power-agent cap clamping + self.power_agent_cap_clamped_total = _FakeCounter() + + # ------------------------------------------------------------------ + # Convenience read helpers for testbed assertions + # ------------------------------------------------------------------ + + def counter_value(self, name: str, **labels: Any) -> float: + obj = getattr(self, name) + if labels: + return obj.labeled_value(**labels) + return obj.value + + def gauge_value(self, name: str, **labels: Any) -> float: + obj = getattr(self, name) + if labels: + return obj.labeled_value(**labels) + return obj.value diff --git a/components/src/dynamo/planner/tests/testbed/fake_prometheus.py b/components/src/dynamo/planner/tests/testbed/fake_prometheus.py new file mode 100644 index 000000000000..6bb18db631d3 --- /dev/null +++ b/components/src/dynamo/planner/tests/testbed/fake_prometheus.py @@ -0,0 +1,126 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""FakePrometheusClient — drop-in for PrometheusAPIClient. + +Shared by both α-class (backed by SyntheticFleet) and γ-class (backed by +SyntheticPowerOverlay). The discriminator is the ``source`` constructor arg. + +The source object must expose ``observation_at(tick: int)`` returning an object +with attributes ``ttft_avg_s``, ``itl_avg_s``, ``power_w_prefill``, +``power_w_decode``, ``total_tokens_per_sec``. +""" + +from __future__ import annotations + +from typing import Any, Optional, Protocol, runtime_checkable + + +@runtime_checkable +class _ObservationSource(Protocol): + def observation_at(self, tick: int) -> Any: + ... + + def is_signal_in_outage(self, signal: str) -> bool: + ... + + def prom_stale_lag(self) -> Optional[int]: + ... + + +class FakePrometheusClient: + """Drop-in for PrometheusAPIClient with same method signatures.""" + + def __init__(self, source: Any) -> None: + self._source = source + self._current_tick: int = 0 + + def set_tick(self, tick: int) -> None: + """Called by runner at start of each tick.""" + self._current_tick = tick + + # ------------------------------------------------------------------ + # Internal helpers + # ------------------------------------------------------------------ + + def _effective_tick(self, signal: str) -> Optional[int]: + """Return the tick to read from (applying stale lag) or None for outage.""" + if self._source.is_signal_in_outage(signal): + return None + lag = self._source.prom_stale_lag() + t = self._current_tick - (lag or 0) + return max(0, t) + + def _get_obs(self, signal: str): + t = self._effective_tick(signal) + if t is None: + return None + return self._source.observation_at(t) + + # ------------------------------------------------------------------ + # PrometheusAPIClient interface (same signatures as the real class) + # ------------------------------------------------------------------ + + def get_avg_time_to_first_token(self, *args: Any, **kwargs: Any) -> Optional[float]: + obs = self._get_obs("ttft") + return obs.ttft_avg_s if obs is not None else None + + def get_avg_inter_token_latency(self, *args: Any, **kwargs: Any) -> Optional[float]: + obs = self._get_obs("itl") + return obs.itl_avg_s if obs is not None else None + + def get_avg_per_gpu_power_by_component( + self, *, component: str, **kwargs: Any + ) -> Optional[float]: + signal = f"power_{component[0]}" # "power_p" or "power_d" + obs = self._get_obs(signal) + if obs is None: + return None + if component == "prefill": + return obs.power_w_prefill + elif component == "decode": + return obs.power_w_decode + return None + + def get_total_dgd_power(self, *args: Any, **kwargs: Any) -> Optional[float]: + obs = self._get_obs("power_p") + if obs is None: + return None + return obs.power_w_prefill + obs.power_w_decode + + def get_avg_request_count(self, *args: Any, **kwargs: Any) -> Optional[float]: + obs = self._get_obs("capacity") + if obs is None: + return None + return float(obs.traffic.num_req) if obs.traffic else 0.0 + + def get_avg_input_sequence_tokens( + self, *args: Any, **kwargs: Any + ) -> Optional[float]: + obs = self._get_obs("capacity") + if obs is None: + return None + return float(obs.traffic.isl) if obs.traffic else 0.0 + + def get_avg_output_sequence_tokens( + self, *args: Any, **kwargs: Any + ) -> Optional[float]: + obs = self._get_obs("capacity") + if obs is None: + return None + return float(obs.traffic.osl) if obs.traffic else 0.0 + + def get_avg_kv_hit_rate(self, *args: Any, **kwargs: Any) -> Optional[float]: + obs = self._get_obs("capacity") + if obs is None: + return None + return obs.traffic.kv_hit_rate if obs.traffic else None + + def get_avg_request_duration(self, *args: Any, **kwargs: Any) -> Optional[float]: + return None + + def warn_if_router_not_scraped(self, *args: Any, **kwargs: Any) -> None: + pass + + def get_recent_and_averaged_metrics(self, *args: Any, **kwargs: Any): # type: ignore[return] + return None diff --git a/components/src/dynamo/planner/tests/testbed/grafana/testbed_dashboard.json b/components/src/dynamo/planner/tests/testbed/grafana/testbed_dashboard.json new file mode 100644 index 000000000000..29d7474694e4 --- /dev/null +++ b/components/src/dynamo/planner/tests/testbed/grafana/testbed_dashboard.json @@ -0,0 +1,414 @@ +{ + "__inputs": [ + { + "name": "DS_PROMETHEUS", + "label": "Prometheus", + "description": "Prometheus data source (node_exporter textfile_collector)", + "type": "datasource", + "pluginId": "prometheus", + "pluginName": "Prometheus" + } + ], + "__requires": [ + { + "type": "grafana", + "id": "grafana", + "name": "Grafana", + "version": "10.0.0" + }, + { + "type": "datasource", + "id": "prometheus", + "name": "Prometheus", + "version": "1.0.0" + }, + { + "type": "panel", + "id": "timeseries", + "name": "Time series", + "version": "" + }, + { + "type": "panel", + "id": "stat", + "name": "Stat", + "version": "" + }, + { + "type": "panel", + "id": "table", + "name": "Table", + "version": "" + } + ], + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { "type": "grafana", "uid": "-- Grafana --" }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "description": "Power Planner Stress Testbed — scenario comparison dashboard. Import Prometheus textfile output with: python -m dynamo.planner.tests.testbed.runner --all --prom-textfile /var/lib/node_exporter/textfile_collector/testbed.prom", + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 1, + "id": null, + "links": [], + "panels": [ + { + "collapsed": false, + "gridPos": { "h": 1, "w": 24, "x": 0, "y": 0 }, + "id": 100, + "title": "Scenario Filter", + "type": "row" + }, + { + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "fieldConfig": { + "defaults": { + "color": { "mode": "palette-classic" }, + "custom": { "hideFrom": { "legend": false, "tooltip": false, "viz": false } }, + "mappings": [] + }, + "overrides": [] + }, + "gridPos": { "h": 8, "w": 24, "x": 0, "y": 1 }, + "id": 1, + "options": { + "legend": { "calcs": ["last"], "displayMode": "table", "placement": "right", "showLegend": true }, + "tooltip": { "mode": "multi", "sort": "none" } + }, + "targets": [ + { + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "expr": "testbed_c_power_d{scenario=~\"$scenario\"}", + "legendFormat": "{{scenario}} c_power_d", + "refId": "A" + }, + { + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "expr": "testbed_c_power_p{scenario=~\"$scenario\"}", + "legendFormat": "{{scenario}} c_power_p", + "refId": "B" + } + ], + "title": "AIC Correction Coefficients (c_power_d / c_power_p)", + "type": "timeseries" + }, + { + "collapsed": false, + "gridPos": { "h": 1, "w": 24, "x": 0, "y": 9 }, + "id": 101, + "title": "Power Budget", + "type": "row" + }, + { + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "fieldConfig": { + "defaults": { + "color": { "mode": "palette-classic" }, + "custom": { "lineWidth": 2 }, + "unit": "watt" + }, + "overrides": [ + { + "matcher": { "id": "byName", "options": "budget_w" }, + "properties": [ + { "id": "custom.lineStyle", "value": { "dash": [8, 4], "fill": "dash" } }, + { "id": "color", "value": { "fixedColor": "red", "mode": "fixed" } } + ] + } + ] + }, + "gridPos": { "h": 8, "w": 12, "x": 0, "y": 10 }, + "id": 2, + "options": { + "legend": { "calcs": ["last", "max"], "displayMode": "table", "placement": "bottom", "showLegend": true }, + "tooltip": { "mode": "multi", "sort": "none" } + }, + "targets": [ + { + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "expr": "testbed_projected_w{scenario=~\"$scenario\"}", + "legendFormat": "{{scenario}} projected_w", + "refId": "A" + }, + { + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "expr": "testbed_budget_w{scenario=~\"$scenario\"}", + "legendFormat": "{{scenario}} budget_w", + "refId": "B" + } + ], + "title": "Projected Power vs Budget", + "type": "timeseries" + }, + { + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "fieldConfig": { + "defaults": { + "color": { "mode": "palette-classic" }, + "custom": { "lineWidth": 2 }, + "unit": "watt" + }, + "overrides": [] + }, + "gridPos": { "h": 8, "w": 12, "x": 12, "y": 10 }, + "id": 3, + "options": { + "legend": { "calcs": ["last"], "displayMode": "table", "placement": "bottom", "showLegend": true }, + "tooltip": { "mode": "multi", "sort": "none" } + }, + "targets": [ + { + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "expr": "testbed_cap_d{scenario=~\"$scenario\"}", + "legendFormat": "{{scenario}} cap_d", + "refId": "A" + }, + { + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "expr": "testbed_cap_p{scenario=~\"$scenario\"}", + "legendFormat": "{{scenario}} cap_p", + "refId": "B" + } + ], + "title": "Applied Power Caps (decode / prefill)", + "type": "timeseries" + }, + { + "collapsed": false, + "gridPos": { "h": 1, "w": 24, "x": 0, "y": 18 }, + "id": 102, + "title": "Replica Counts", + "type": "row" + }, + { + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "fieldConfig": { + "defaults": { + "color": { "mode": "palette-classic" }, + "custom": { "lineWidth": 2 }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { "h": 8, "w": 12, "x": 0, "y": 19 }, + "id": 4, + "options": { + "legend": { "calcs": ["last"], "displayMode": "table", "placement": "bottom", "showLegend": true }, + "tooltip": { "mode": "multi", "sort": "none" } + }, + "targets": [ + { + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "expr": "testbed_n_d{scenario=~\"$scenario\"}", + "legendFormat": "{{scenario}} n_d", + "refId": "A" + }, + { + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "expr": "testbed_n_p{scenario=~\"$scenario\"}", + "legendFormat": "{{scenario}} n_p", + "refId": "B" + } + ], + "title": "Replica Counts (decode / prefill)", + "type": "timeseries" + }, + { + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "fieldConfig": { + "defaults": { + "color": { "mode": "palette-classic" }, + "custom": { "lineWidth": 1 }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { "h": 8, "w": 12, "x": 12, "y": 19 }, + "id": 5, + "options": { + "legend": { "calcs": ["sum"], "displayMode": "table", "placement": "bottom", "showLegend": true }, + "tooltip": { "mode": "multi", "sort": "none" } + }, + "targets": [ + { + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "expr": "testbed_sweep_fired{scenario=~\"$scenario\"}", + "legendFormat": "{{scenario}} sweep_fired", + "refId": "A" + }, + { + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "expr": "testbed_sla_violated{scenario=~\"$scenario\"}", + "legendFormat": "{{scenario}} sla_violated", + "refId": "B" + } + ], + "title": "Events: Sweep Fired / SLA Violated", + "type": "timeseries" + }, + { + "collapsed": false, + "gridPos": { "h": 1, "w": 24, "x": 0, "y": 27 }, + "id": 103, + "title": "Observed Metrics", + "type": "row" + }, + { + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "fieldConfig": { + "defaults": { + "color": { "mode": "palette-classic" }, + "unit": "watt" + }, + "overrides": [] + }, + "gridPos": { "h": 8, "w": 12, "x": 0, "y": 28 }, + "id": 6, + "options": { + "legend": { "calcs": ["mean", "max"], "displayMode": "table", "placement": "bottom", "showLegend": true }, + "tooltip": { "mode": "multi", "sort": "none" } + }, + "targets": [ + { + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "expr": "testbed_observed_power_w_d{scenario=~\"$scenario\"}", + "legendFormat": "{{scenario}} observed decode W", + "refId": "A" + }, + { + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "expr": "testbed_observed_power_w_p{scenario=~\"$scenario\"}", + "legendFormat": "{{scenario}} observed prefill W", + "refId": "B" + } + ], + "title": "Observed GPU Power (decode / prefill)", + "type": "timeseries" + }, + { + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "fieldConfig": { + "defaults": { + "color": { "mode": "palette-classic" }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { "h": 8, "w": 12, "x": 12, "y": 28 }, + "id": 7, + "options": { + "legend": { "calcs": ["mean"], "displayMode": "table", "placement": "bottom", "showLegend": true }, + "tooltip": { "mode": "multi", "sort": "none" } + }, + "targets": [ + { + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "expr": "testbed_observed_ttft_s{scenario=~\"$scenario\"}", + "legendFormat": "{{scenario}} TTFT", + "refId": "A" + }, + { + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "expr": "testbed_observed_itl_s{scenario=~\"$scenario\"}", + "legendFormat": "{{scenario}} ITL", + "refId": "B" + } + ], + "title": "Observed Latency (TTFT / ITL)", + "type": "timeseries" + }, + { + "collapsed": false, + "gridPos": { "h": 1, "w": 24, "x": 0, "y": 36 }, + "id": 104, + "title": "Scenario Summary", + "type": "row" + }, + { + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "align": "auto", + "displayMode": "auto" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { "color": "green", "value": null }, + { "color": "red", "value": 1 } + ] + } + }, + "overrides": [] + }, + "gridPos": { "h": 8, "w": 24, "x": 0, "y": 37 }, + "id": 8, + "options": { + "footer": { "enablePagination": false, "fields": "", "reducer": ["sum"], "show": false }, + "showHeader": true, + "sortBy": [{ "desc": false, "displayName": "scenario" }] + }, + "targets": [ + { + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "expr": "max by (scenario) (testbed_optimizer_exceptions{scenario=~\"$scenario\"})", + "legendFormat": "{{scenario}}", + "refId": "A" + } + ], + "title": "Optimizer Exceptions per Scenario", + "transformations": [ + { "id": "labelsToFields", "options": {} }, + { "id": "merge", "options": {} } + ], + "type": "table" + } + ], + "refresh": "", + "schemaVersion": 37, + "style": "dark", + "tags": ["dynamo", "power-planner", "testbed"], + "templating": { + "list": [ + { + "current": { "selected": true, "text": "All", "value": "$__all" }, + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "definition": "label_values(testbed_projected_w, scenario)", + "hide": 0, + "includeAll": true, + "multi": true, + "name": "scenario", + "options": [], + "query": { + "query": "label_values(testbed_projected_w, scenario)", + "refId": "StandardVariableQuery" + }, + "refresh": 1, + "regex": "", + "sort": 1, + "type": "query", + "label": "Scenario" + } + ] + }, + "time": { "from": "now-1h", "to": "now" }, + "timepicker": {}, + "timezone": "browser", + "title": "Power Planner Stress Testbed", + "uid": "dynamo-pp-testbed-v1", + "version": 1 +} diff --git a/components/src/dynamo/planner/tests/testbed/recorder.py b/components/src/dynamo/planner/tests/testbed/recorder.py new file mode 100644 index 000000000000..f046bfd49e46 --- /dev/null +++ b/components/src/dynamo/planner/tests/testbed/recorder.py @@ -0,0 +1,194 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""TickRecorder / TickHistory — per-tick snapshot store. + +Records one TickSnapshot per tick; serialises to CSV and Prometheus textfile. +Optional plot output requires ``matplotlib`` (install dynamo[testbed-plot]). +""" + +from __future__ import annotations + +import csv +import dataclasses +from dataclasses import dataclass +from pathlib import Path +from typing import Optional + + +@dataclass +class TickSnapshot: + """One tick's worth of recorded data.""" + + tick: int + # Applied state: + n_p: int + n_d: int + cap_p: int + cap_d: int + # Observed (post-noise / post-overlay): + observed_ttft_s: float + observed_itl_s: float + observed_power_w_p: float + observed_power_w_d: float + observed_capacity_tps: float + # Controller state: + c_ttft: float + c_itl: float + c_power_p: float + c_power_d: float + estimated_throughput: float + consecutive_violation_ticks: int + # Aggregate: + projected_w: float + budget_w: float + # Events fired this tick: + sweep_fired: bool + sla_violated: bool + capacity_exceeded: bool + # Counters delta: + cap_clamped_min: int + cap_clamped_max: int + optimizer_exceptions: int + correction_pegged: dict[str, int] = dataclasses.field(default_factory=dict) + admission_partial_failures: int = 0 + # Cumulative count of replica direction flips (n_p sign change OR n_d sign + # change relative to the previous tick's delta). Useful for asserting + # "no oscillation" in scale-up/scale-down scenarios. + n_oscillations: int = 0 + # γ-only columns (None in α): + mocker_active_p: Optional[int] = None + mocker_active_d: Optional[int] = None + mocker_kv_hit_rate: Optional[float] = None + + # Alias properties so assertions can use short names + @property + def c_power_prefill(self) -> float: + return self.c_power_p + + @property + def c_power_decode(self) -> float: + return self.c_power_d + + +# Fields visible to assertion DSL (validated at load time) +TICK_SNAPSHOT_FIELDS = {f.name for f in dataclasses.fields(TickSnapshot)} + + +class TickHistory: + """Container for all tick snapshots from a scenario run.""" + + def __init__(self) -> None: + self.snapshots: list[TickSnapshot] = [] + + def append(self, snap: TickSnapshot) -> None: + self.snapshots.append(snap) + + def __len__(self) -> int: + return len(self.snapshots) + + def __getitem__(self, idx: int) -> TickSnapshot: + return self.snapshots[idx] + + # ------------------------------------------------------------------ + # Serialisation + # ------------------------------------------------------------------ + + def to_csv(self, path: Path) -> None: + path = Path(path) + path.parent.mkdir(parents=True, exist_ok=True) + fields = [ + f.name + for f in dataclasses.fields(TickSnapshot) + if f.name not in ("correction_pegged",) + ] + with path.open("w", newline="") as f: + writer = csv.DictWriter(f, fieldnames=fields + ["correction_pegged_json"]) + writer.writeheader() + import json + + for snap in self.snapshots: + row = {k: getattr(snap, k) for k in fields} + row["correction_pegged_json"] = json.dumps(snap.correction_pegged) + writer.writerow(row) + + def to_prom_textfile(self, path: Path, scenario_name: str = "unknown") -> None: + """Emit a Prometheus textfile-collector-compatible .prom file.""" + path = Path(path) + path.parent.mkdir(parents=True, exist_ok=True) + labels = f'scenario="{scenario_name}"' + lines = [] + if self.snapshots: + last = self.snapshots[-1] + lines += [ + f"testbed_c_ttft{{{labels}}} {last.c_ttft}", + f"testbed_c_itl{{{labels}}} {last.c_itl}", + f"testbed_c_power_p{{{labels}}} {last.c_power_p}", + f"testbed_c_power_d{{{labels}}} {last.c_power_d}", + f"testbed_cap_p_watts{{{labels}}} {last.cap_p}", + f"testbed_cap_d_watts{{{labels}}} {last.cap_d}", + f"testbed_projected_w{{{labels}}} {last.projected_w}", + f"testbed_budget_w{{{labels}}} {last.budget_w}", + f"testbed_n_p{{{labels}}} {last.n_p}", + f"testbed_n_d{{{labels}}} {last.n_d}", + ] + path.write_text("\n".join(lines) + "\n") + + def plot(self, path: Path, scenario_name: str = "unknown") -> None: + """Emit a multi-panel PNG plot (requires matplotlib).""" + try: + import matplotlib.pyplot as plt + except ImportError: + raise ImportError( + "matplotlib is required for plot output. " + "Install with: pip install dynamo[testbed-plot]" + ) + ticks = [s.tick for s in self.snapshots] + fig, axes = plt.subplots(3, 2, figsize=(14, 10)) + fig.suptitle(f"Testbed scenario: {scenario_name}", fontsize=12) + + axes[0, 0].plot(ticks, [s.c_power_p for s in self.snapshots], label="c_power_p") + axes[0, 0].plot(ticks, [s.c_power_d for s in self.snapshots], label="c_power_d") + axes[0, 0].axhline(1.0, color="gray", linestyle="--", alpha=0.5) + axes[0, 0].set_title("Power correction coefficients") + axes[0, 0].legend() + + axes[0, 1].plot(ticks, [s.cap_p for s in self.snapshots], label="cap_p (W)") + axes[0, 1].plot(ticks, [s.cap_d for s in self.snapshots], label="cap_d (W)") + axes[0, 1].set_title("Applied caps (W/GPU)") + axes[0, 1].legend() + + axes[1, 0].plot( + ticks, [s.projected_w for s in self.snapshots], label="projected_w" + ) + axes[1, 0].plot( + ticks, + [s.budget_w for s in self.snapshots], + label="budget_w", + linestyle="--", + ) + axes[1, 0].set_title("Power budget utilization") + axes[1, 0].legend() + + axes[1, 1].plot( + ticks, [s.observed_ttft_s * 1000 for s in self.snapshots], label="TTFT (ms)" + ) + axes[1, 1].plot( + ticks, [s.observed_itl_s * 1000 for s in self.snapshots], label="ITL (ms)" + ) + axes[1, 1].set_title("Observed latency") + axes[1, 1].legend() + + axes[2, 0].plot(ticks, [s.n_p for s in self.snapshots], label="n_p") + axes[2, 0].plot(ticks, [s.n_d for s in self.snapshots], label="n_d") + axes[2, 0].set_title("Replica counts") + axes[2, 0].legend() + + axes[2, 1].plot(ticks, [s.observed_capacity_tps for s in self.snapshots]) + axes[2, 1].set_title("Observed capacity (tok/s)") + + plt.tight_layout() + path = Path(path) + path.parent.mkdir(parents=True, exist_ok=True) + plt.savefig(path, dpi=100) + plt.close(fig) diff --git a/components/src/dynamo/planner/tests/testbed/replay/__init__.py b/components/src/dynamo/planner/tests/testbed/replay/__init__.py new file mode 100644 index 000000000000..1ed4356ee8fa --- /dev/null +++ b/components/src/dynamo/planner/tests/testbed/replay/__init__.py @@ -0,0 +1,3 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""γ-class testbed components (extends ReplayPlannerAdapter with power signals).""" diff --git a/components/src/dynamo/planner/tests/testbed/replay/power_aware_replay_adapter.py b/components/src/dynamo/planner/tests/testbed/replay/power_aware_replay_adapter.py new file mode 100644 index 000000000000..9a9abafed321 --- /dev/null +++ b/components/src/dynamo/planner/tests/testbed/replay/power_aware_replay_adapter.py @@ -0,0 +1,518 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""PowerAwareReplayAdapter — γ-class subclass of ReplayPlannerAdapter. + +Extends the existing replay loop with: + 1. SyntheticPowerOverlay.observe() after bridge.advance_to() + 2. AICPowerOptimizer.update_correction() + should_reoptimize() / optimize() + 3. state_machine._apply_power_budget() after on_tick() + 4. Feeding into TickHistory (via the testbed TickRecorder) + +Does NOT modify ReplayPlannerAdapter itself — pure subclass extension. +""" + +from __future__ import annotations + +import logging +from typing import TYPE_CHECKING, Any, Optional + +from dynamo.planner.offline.replay_adapter import ReplayPlannerAdapter + +if TYPE_CHECKING: + from dynamo.planner.config.planner_config import PlannerConfig + from dynamo.planner.core.types import WorkerCapabilities + from dynamo.planner.tests.testbed.fake_aic import FakeAIC + from dynamo.planner.tests.testbed.fake_planner_metrics import FakePlannerMetrics + from dynamo.planner.tests.testbed.fake_prometheus import FakePrometheusClient + from dynamo.planner.tests.testbed.recorder import TickHistory + from dynamo.planner.tests.testbed.replay.replay_fake_actuator import ( + ReplayFakeActuator, + ) + from dynamo.planner.tests.testbed.replay.synthetic_power_overlay import ( + SyntheticPowerOverlay, + ) + from dynamo.planner.tests.testbed.scenarios import ScenarioSpec + +logger = logging.getLogger(__name__) + + +class PowerAwareReplayAdapter(ReplayPlannerAdapter): + """Extends ReplayPlannerAdapter with the power-aware control loop. + + Does not modify the parent class; overrides ``run()`` only to interleave + new steps between ``bridge.advance_to()`` and ``bridge.apply_scaling()``. + """ + + def __init__( + self, + planner_config: "PlannerConfig", + bridge: Any, + scenario: "ScenarioSpec", + *, + overlay: "SyntheticPowerOverlay", + fake_prom: "FakePrometheusClient", + fake_aic: "FakeAIC", + actuator: "ReplayFakeActuator", + metrics: "FakePlannerMetrics", + capabilities: Optional["WorkerCapabilities"] = None, + warmup_observations: Optional[list] = None, + ) -> None: + super().__init__(planner_config, bridge, capabilities, warmup_observations) + self._overlay = overlay + self._fake_prom = fake_prom + self._fake_aic = fake_aic + self._actuator = actuator + self._metrics = metrics + self._scenario = scenario + + from dynamo.planner.monitoring.aic_power_optimizer import AICPowerOptimizer + + if planner_config.enable_aic_optimizer: + self._optimizer: Optional[AICPowerOptimizer] = AICPowerOptimizer( + planner_config, metrics + ) + self._optimizer._aic_estimator_factory = fake_aic.make_estimator_factory() + # Initial sweep + initial = self._optimizer.optimize() + if initial is not None: + planner_config.prefill_engine_gpu_power_limit = initial.cap_p + planner_config.decode_engine_gpu_power_limit = initial.cap_d + self._optimizer._estimated_throughput = ( + initial.aic_seq_per_s_per_replica + * initial.n_d + * (initial.isl + initial.osl) + ) + else: + self._optimizer = None + + self._tick_history: Optional["TickHistory"] = None + + def _attach_history(self, history: "TickHistory") -> None: + self._tick_history = history + + def run(self): # type: ignore[override] + """Run γ-class replay loop with power-aware extensions.""" + from dynamo.planner.offline.replay_adapter import ReplayPlannerReport + + next_tick = self._sm.initial_tick(0.0) + scaling_events = [] + diagnostics_log = [] + total_ticks = 0 + + while True: + tick_ms = next_tick.at_s * 1000.0 + result = self._bridge.advance_to(tick_ms) + + if result["is_done"]: + break + + # Compute virtual tick index + vtick = int( + next_tick.at_s / self._config.throughput_adjustment_interval_seconds + ) + self._fake_prom.set_tick(vtick) + + # --- γ extension: overlay synthesises power from FPMs --- + # The Rust bridge returns prefill/decode snapshots in separate + # lists; tag each and concatenate so the overlay (which is + # component-aware) can dispatch on snap["component"]. + prefill_snaps = result.get("prefill_fpm_snapshots", []) + decode_snaps = result.get("decode_fpm_snapshots", []) + for s in prefill_snaps: + s["component"] = "prefill" + for s in decode_snaps: + s["component"] = "decode" + self._overlay.observe( + fpm_snapshots=prefill_snaps + decode_snaps, + applied_caps=self._actuator.applied_caps_snapshot(), + tick=vtick, + ) + + tick_input = self._build_tick_input(next_tick, result) + + # --- γ extension: optimizer correction + drift check --- + if self._optimizer is not None and tick_input.traffic is not None: + prom_power_p = self._fake_prom.get_avg_per_gpu_power_by_component( + component="prefill", interval="60s" + ) + prom_power_d = self._fake_prom.get_avg_per_gpu_power_by_component( + component="decode", interval="60s" + ) + self._optimizer.update_correction( + traffic=tick_input.traffic, + observed_ttft_avg=( + tick_input.traffic.ttft_avg if tick_input.traffic else None + ), + observed_itl_avg=( + tick_input.traffic.itl_avg if tick_input.traffic else None + ), + observed_power_w_prefill=prom_power_p, + observed_power_w_decode=prom_power_d, + ) + if self._optimizer.should_reoptimize(tick_input.traffic): + new_cfg = self._optimizer.optimize() + if new_cfg is not None: + self._config.prefill_engine_gpu_power_limit = new_cfg.cap_p + self._config.decode_engine_gpu_power_limit = new_cfg.cap_d + self._actuator.apply_caps(new_cfg.cap_p, new_cfg.cap_d) + self._optimizer._estimated_throughput = ( + new_cfg.aic_seq_per_s_per_replica + * new_cfg.n_d + * (new_cfg.isl + new_cfg.osl) + ) + + # --- Existing: state machine tick --- + effects = self._sm.on_tick(next_tick, tick_input) + diagnostics_log.append(effects.diagnostics) + total_ticks += 1 + + self._record_diagnostics(tick_input, effects, result) + + active_p = result["active_prefill_count"] + active_d = result["active_decode_count"] + if ( + self._scaling_target_prefill is not None + and active_p == self._scaling_target_prefill + ): + self._scaling_target_prefill = None + if ( + self._scaling_target_decode is not None + and active_d == self._scaling_target_decode + ): + self._scaling_target_decode = None + + # --- γ extension: power budget post-clamp --- + if effects.scale_to is not None and self._config.enable_power_awareness: + clamped_p, clamped_d = self._sm._apply_power_budget( + effects.scale_to.num_prefill or active_p, + effects.scale_to.num_decode or active_d, + ) + effects.scale_to.num_prefill = clamped_p + effects.scale_to.num_decode = clamped_d + + if effects.scale_to is not None: + self._apply_scaling(effects, result, tick_input.now_s, scaling_events) + + if effects.next_tick is None: + break + next_tick = effects.next_tick + + trace_report = self._bridge.finalize() + html_report_path = self._recorder.finalize() + return ReplayPlannerReport( + trace_report=trace_report, + scaling_events=scaling_events, + diagnostics_log=diagnostics_log, + total_ticks=total_ticks, + html_report_path=html_report_path, + ) + + +# --------------------------------------------------------------------------- +# Builder +# --------------------------------------------------------------------------- + + +def build_gamma_adapter(scenario: "ScenarioSpec") -> "GammaHarness": + """Construct the full γ-class harness from a scenario spec.""" + return GammaHarness(scenario) + + +class GammaHarness: + """Container for the full γ-class setup.""" + + def __init__(self, scenario: "ScenarioSpec") -> None: + self.scenario = scenario + self._setup() + + def _setup(self) -> None: + import random + + from dynamo.planner.config.aic_interpolation_spec import AICInterpolationSpec + from dynamo.planner.config.parallelization import PickedParallelConfig + from dynamo.planner.config.planner_config import PlannerConfig + from dynamo.planner.core.types import EngineCapabilities, WorkerCapabilities + from dynamo.planner.tests.testbed.fake_aic import FakeAIC + from dynamo.planner.tests.testbed.fake_planner_metrics import FakePlannerMetrics + from dynamo.planner.tests.testbed.fake_prometheus import FakePrometheusClient + from dynamo.planner.tests.testbed.replay.replay_fake_actuator import ( + ReplayFakeActuator, + ) + from dynamo.planner.tests.testbed.replay.synthetic_power_overlay import ( + SyntheticPowerOverlay, + ) + from dynamo.planner.tests.testbed.scenarios import SystemSpec + + sc = self.scenario + rng = random.Random(sc.seed) + mocker = sc.mocker + overlay_spec = sc.overlay + planner_spec = sc.planner + + system_spec = SystemSpec.load(overlay_spec.system) + + aic_spec = AICInterpolationSpec( + hf_id="fake-model/testbed", + system=overlay_spec.system, + backend="vllm", + isl=3000, + osl=150, + sweep_max_context_length=8192, + prefill_interpolation_granularity=1, + decode_interpolation_granularity=1, + prefill_pick=PickedParallelConfig(tp=1, pp=1, dp=1), + decode_pick=PickedParallelConfig(tp=1, pp=1, dp=1), + ) + self.config = PlannerConfig( + mode=planner_spec.mode, + ttft=planner_spec.ttft, + itl=planner_spec.itl, + enable_power_awareness=planner_spec.enable_power_awareness, + enable_aic_optimizer=planner_spec.enable_aic_optimizer, + total_gpu_power_limit=planner_spec.total_gpu_power_limit, + power_agent_safe_default_watts=planner_spec.power_agent_safe_default_watts, + prefill_engine_gpu_power_limit=planner_spec.prefill_engine_gpu_power_limit, + decode_engine_gpu_power_limit=planner_spec.decode_engine_gpu_power_limit, + aic_initial_c_power_prefill=planner_spec.aic_initial_c_power_prefill, + aic_initial_c_power_decode=planner_spec.aic_initial_c_power_decode, + aic_initial_c_power_agg=planner_spec.aic_initial_c_power_agg, + aic_initial_c_ttft=planner_spec.aic_initial_c_ttft, + aic_initial_c_itl=planner_spec.aic_initial_c_itl, + aic_reoptimize_interval=planner_spec.aic_reoptimize_interval, + aic_drift_relative_threshold=planner_spec.aic_drift_relative_threshold, + aic_drift_consecutive_ticks=planner_spec.aic_drift_consecutive_ticks, + aic_max_consecutive_failures=planner_spec.aic_max_consecutive_failures, + min_endpoint=planner_spec.min_endpoint, + max_gpu_budget=planner_spec.max_gpu_budget, + aic_interpolation=aic_spec, + live_dashboard_port=0, + report_interval_hours=None, + ) + + caps = WorkerCapabilities( + prefill=EngineCapabilities(num_gpu=1), + decode=EngineCapabilities( + num_gpu=1, + max_kv_tokens=system_spec.max_kv_tokens, + ), + ) + + self.metrics = FakePlannerMetrics() + self.fake_aic = FakeAIC(system_spec) + self.overlay = SyntheticPowerOverlay(overlay_spec, system_spec, sc, rng) + self.prom = FakePrometheusClient(source=self.overlay) + + # Build the bridge + self.bridge = self._build_bridge(mocker, overlay_spec.system) + + self.actuator = ReplayFakeActuator( + sc, self.overlay, self.bridge, self.metrics, system_spec + ) + + self.adapter = PowerAwareReplayAdapter( + self.config, + self.bridge, + sc, + overlay=self.overlay, + fake_prom=self.prom, + fake_aic=self.fake_aic, + actuator=self.actuator, + metrics=self.metrics, + capabilities=caps, + ) + + def _build_bridge(self, mocker, system: str) -> Any: + """Build the PlannerReplayBridge from mocker spec. + + Falls back to a minimal stub when the Rust bridge is unavailable + (e.g. in unit tests that don't compile the extension). + + Handles two generations of the PlannerReplayBridge API: + - Newer bindings: from_trace_file_disagg / from_synthetic_disagg + - Older bindings (installed on dev pods): create_disagg(trace_file, ...) + with no synthetic-workload constructor; falls back to placeholder trace. + """ + import pathlib + + try: + from dynamo.llm import PlannerReplayBridge # type: ignore[import] + except ImportError: + return _StubBridge() + + if mocker is None or (not mocker.trace_file and not mocker.synthetic_workload): + return _StubBridge() + + _has_trace_api = hasattr(PlannerReplayBridge, "from_trace_file_disagg") + _has_synthetic = hasattr(PlannerReplayBridge, "from_synthetic_disagg") + _has_create_disagg = hasattr(PlannerReplayBridge, "create_disagg") + + trace_path = mocker.trace_file + + if not trace_path and not _has_synthetic: + # Older binding — no synthetic workload constructor. + # Fall back to the placeholder trace bundled with the testbed. + placeholder = ( + pathlib.Path(__file__).parent.parent + / "traces" + / "placeholder_h200_disagg_1rps.jsonl" + ) + if not placeholder.exists(): + return _StubBridge() + trace_path = str(placeholder) + + if trace_path: + if _has_trace_api: + return PlannerReplayBridge.from_trace_file_disagg( + trace_path=trace_path, + num_prefill_workers=mocker.num_prefill_workers, + num_decode_workers=mocker.num_decode_workers, + trace_block_size=mocker.trace_block_size, + arrival_speedup_ratio=mocker.arrival_speedup_ratio, + router_mode=mocker.router_mode, + prefill_engine_args=mocker.prefill_engine_args, + decode_engine_args=mocker.decode_engine_args, + ) + # _has_create_disagg — older API (trace_file positional, no trace_path kw). + # engine_args must be MockEngineArgs objects, not plain dicts. + # speedup_ratio=1000 avoids real-time simulation (default=1.0 → 3600s wall + # time for a 60-tick × 60s scenario — unsuitable for a test suite). + from dynamo._core import MockEngineArgs # type: ignore[import] + + def _to_engine_args(d: dict) -> "MockEngineArgs": + return MockEngineArgs( + block_size=d.get("block_size", 0), + max_num_batched_tokens=d.get("max_num_batched_tokens"), + max_num_seqs=d.get("max_num_seqs"), + speedup_ratio=100.0, + decode_speedup_ratio=100.0, + ) + + # Use 1 worker each to keep active_count == FPM-reported count. + # The state machine reconcile loop spins when the bridge reports + # active_decode_count=N but FPMs only show 1 worker (placeholder + # trace has no multi-worker KV traffic to distribute to N workers). + return PlannerReplayBridge.create_disagg( + trace_file=trace_path, + prefill_engine_args=_to_engine_args(mocker.prefill_engine_args), + decode_engine_args=_to_engine_args(mocker.decode_engine_args), + num_prefill_workers=1, + num_decode_workers=1, + router_mode="round_robin", + arrival_speedup_ratio=mocker.arrival_speedup_ratio, + trace_block_size=mocker.trace_block_size, + ) + + # Newer binding synthetic workload path + return PlannerReplayBridge.from_synthetic_disagg( + num_prefill_workers=mocker.num_prefill_workers, + num_decode_workers=mocker.num_decode_workers, + arrival_rate=mocker.arrival_rate, + isl=mocker.isl, + osl=mocker.osl, + ) + + def run_and_record(self) -> "TickHistory": + from dynamo.planner.tests.testbed.recorder import TickHistory, TickSnapshot + + report = self.adapter.run() + + # γ history is built from the diagnostics log. Cap/replica state are + # read from the adapter's recorder snapshots so n_oscillations can be + # computed consistently with α. + history = TickHistory() + opt = self.adapter._optimizer + prev_n_p: Optional[int] = None + prev_n_d: Optional[int] = None + last_dp = 0 + last_dd = 0 + osc = 0 + for i, diag in enumerate(report.diagnostics_log): + n_p = self.adapter._actuator._current_n_p + n_d = self.adapter._actuator._current_n_d + if prev_n_p is not None and prev_n_d is not None: + dp = n_p - prev_n_p + dd = n_d - prev_n_d + if dp != 0: + if last_dp != 0 and (dp > 0) != (last_dp > 0): + osc += 1 + last_dp = dp + if dd != 0: + if last_dd != 0 and (dd > 0) != (last_dd > 0): + osc += 1 + last_dd = dd + prev_n_p, prev_n_d = n_p, n_d + + snap = TickSnapshot( + tick=i, + n_p=n_p, + n_d=n_d, + cap_p=self.config.prefill_engine_gpu_power_limit, + cap_d=self.config.decode_engine_gpu_power_limit, + observed_ttft_s=0.0, + observed_itl_s=0.0, + observed_power_w_p=0.0, + observed_power_w_d=0.0, + observed_capacity_tps=0.0, + c_ttft=opt._c_ttft if opt else 1.0, + c_itl=opt._c_itl if opt else 1.0, + c_power_p=opt._c_power_p if opt else 1.0, + c_power_d=opt._c_power_d if opt else 1.0, + estimated_throughput=opt._estimated_throughput if opt else 0.0, + consecutive_violation_ticks=opt._consecutive_violation_ticks + if opt + else 0, + projected_w=0.0, + budget_w=float(self.config.total_gpu_power_limit or 0), + sweep_fired=False, + sla_violated=False, + capacity_exceeded=False, + cap_clamped_min=int( + self.metrics.power_agent_cap_clamped_total.labeled_value( + direction="min" + ) + ), + cap_clamped_max=int( + self.metrics.power_agent_cap_clamped_total.labeled_value( + direction="max" + ) + ), + optimizer_exceptions=int( + self.metrics.aic_optimizer_exceptions_total.value + ), + admission_partial_failures=int( + self.metrics.admission_partial_success_total.value + ), + n_oscillations=osc, + mocker_active_p=n_p, + mocker_active_d=n_d, + ) + history.append(snap) + return history + + +class _StubBridge: + """Minimal stub when the Rust bridge is unavailable. + + Emits a single done tick immediately so γ tests that lack the mocker + extension don't crash — they just run 0 ticks and skip assertions. + """ + + def advance_to(self, tick_ms: float) -> dict: + return { + "is_done": True, + "now_ms": tick_ms, + "active_prefill_count": 1, + "active_decode_count": 4, + "prefill_fpm_snapshots": [], + "decode_fpm_snapshots": [], + } + + def apply_scaling(self, n_p: int, n_d: int) -> None: + pass + + def drain_traffic(self) -> dict: + return {"duration_s": 0.0, "num_req": 0} + + def finalize(self) -> dict: + return {} diff --git a/components/src/dynamo/planner/tests/testbed/replay/replay_fake_actuator.py b/components/src/dynamo/planner/tests/testbed/replay/replay_fake_actuator.py new file mode 100644 index 000000000000..f13fae06d507 --- /dev/null +++ b/components/src/dynamo/planner/tests/testbed/replay/replay_fake_actuator.py @@ -0,0 +1,150 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""ReplayFakeActuator — γ-class actuator. + +Subclass of FakeActuator that wraps ``PlannerReplayBridge.apply_scaling()`` +for replica changes. Cap annotations and frontend POST faults behave +identically to the α-class parent. + +Reflects applied caps back into SyntheticPowerOverlay so the truth model +uses the actually-applied (possibly clamped) cap. +""" + +from __future__ import annotations + +import random +from typing import TYPE_CHECKING, Any, Optional + +from dynamo.planner.config.defaults import SubComponentType +from dynamo.planner.tests.testbed.fake_actuator import AppliedCaps, FakeActuator + +if TYPE_CHECKING: + from dynamo.planner.tests.testbed.fake_planner_metrics import FakePlannerMetrics + from dynamo.planner.tests.testbed.replay.synthetic_power_overlay import ( + SyntheticPowerOverlay, + ) + from dynamo.planner.tests.testbed.scenarios import ScenarioSpec, SystemSpec + + +class ReplayFakeActuator(FakeActuator): + """γ-class actuator. + + Wraps PlannerReplayBridge.apply_scaling for replica changes; otherwise + identical fault-injection behaviour to FakeActuator. + """ + + def __init__( + self, + scenario: "ScenarioSpec", + overlay: "SyntheticPowerOverlay", + bridge: Any, # PlannerReplayBridge (Rust pyclass) + metrics: "FakePlannerMetrics", + system_spec: "SystemSpec", + ) -> None: + # No SyntheticFleet in γ-class — overlay is the truth source. + # We pass fleet=None but override the methods that use it. + super().__init__( + scenario=scenario, + fleet=None, # type: ignore[arg-type] + metrics=metrics, + system_spec=system_spec, + ) + self._bridge = bridge + self._overlay = overlay + + self._current_n_p = ( + scenario.mocker.num_prefill_workers if scenario.mocker else 1 + ) + self._current_n_d = scenario.mocker.num_decode_workers if scenario.mocker else 4 + + # γ-class has no SyntheticFleet, so the FakeActuator parent's + # post_busy_threshold() (which reads self._fleet.frontend_fault()) is + # not safe — we override below and store the active fault locally. + self._frontend_fault_fraction: Optional[float] = None + + # ------------------------------------------------------------------ + # Override: replica scaling goes through bridge + # ------------------------------------------------------------------ + + async def add_component( + self, sub_component_type: SubComponentType, blocking: bool = True + ) -> None: + if self._actuation_fault_mode == "rbac_denied": + raise RuntimeError("403 Forbidden (synthetic actuation fault)") + if sub_component_type.name.lower() == "prefill": + self._current_n_p += 1 + else: + self._current_n_d += 1 + self._bridge.apply_scaling(self._current_n_p, self._current_n_d) + + async def remove_component( + self, sub_component_type: SubComponentType, blocking: bool = True + ) -> None: + if self._actuation_fault_mode == "rbac_denied": + raise RuntimeError("403 Forbidden (synthetic actuation fault)") + if sub_component_type.name.lower() == "prefill": + self._current_n_p = max(0, self._current_n_p - 1) + else: + self._current_n_d = max(0, self._current_n_d - 1) + self._bridge.apply_scaling(self._current_n_p, self._current_n_d) + + def apply_replicas(self, n_p: int, n_d: int) -> None: + if self._actuation_fault_mode == "rbac_denied": + raise RuntimeError("403 Forbidden (synthetic actuation fault)") + self._current_n_p = max(0, n_p) + self._current_n_d = max(0, n_d) + self._bridge.apply_scaling(self._current_n_p, self._current_n_d) + + # ------------------------------------------------------------------ + # Override: cap reflection goes into overlay (not fleet) + # ------------------------------------------------------------------ + + def apply_caps(self, cap_p: int, cap_d: int) -> None: + clamped_p = self._clamp_raw(cap_p) + clamped_d = self._clamp_raw(cap_d) + if clamped_p > cap_p or clamped_d > cap_d: + self._metrics.power_agent_cap_clamped_total.labels(direction="min").inc() + if clamped_p < cap_p or clamped_d < cap_d: + self._metrics.power_agent_cap_clamped_total.labels(direction="max").inc() + self._applied_cap_p = clamped_p + self._applied_cap_d = clamped_d + self._overlay.notify_caps_changed(clamped_p, clamped_d) + + def patch_pod_annotation(self, pod_name: str, key: str, value: str) -> None: + if self._actuation_fault_mode == "rbac_denied": + raise RuntimeError("403 Forbidden (synthetic actuation fault)") + if pod_name not in self._annotations: + self._annotations[pod_name] = {} + self._annotations[pod_name][key] = value + if self._actuation_fault_mode != "daemonset_absent": + try: + cap_w = int(value) + clamped = self._clamp_raw(cap_w) + if "prefill" in key.lower(): + self._overlay.notify_caps_changed(clamped, self._applied_cap_d) + else: + self._overlay.notify_caps_changed(self._applied_cap_p, clamped) + except (ValueError, TypeError): + pass + + def applied_caps_snapshot(self) -> AppliedCaps: + return AppliedCaps(cap_p=self._applied_cap_p, cap_d=self._applied_cap_d) + + # ------------------------------------------------------------------ + # Override: frontend POST fault state lives on the actuator (no fleet) + # ------------------------------------------------------------------ + + def set_frontend_fault_fraction(self, fraction: Optional[float]) -> None: + """Called by the γ-runner when a FrontendPostFaultEvent fires/expires.""" + self._frontend_fault_fraction = fraction + + async def post_busy_threshold( + self, pod: str, model: str, port: int, **thresholds: float + ) -> None: + frac = self._frontend_fault_fraction + if frac is not None and random.random() < frac: + self._metrics.admission_partial_success_total.inc() + raise RuntimeError( + f"503 Service Unavailable (synthetic POST fault to {pod})" + ) diff --git a/components/src/dynamo/planner/tests/testbed/replay/synthetic_power_overlay.py b/components/src/dynamo/planner/tests/testbed/replay/synthetic_power_overlay.py new file mode 100644 index 000000000000..b4a4cddb00af --- /dev/null +++ b/components/src/dynamo/planner/tests/testbed/replay/synthetic_power_overlay.py @@ -0,0 +1,217 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""SyntheticPowerOverlay — γ-class power signal synthesiser. + +Reads mocker FPM snapshots, computes per-component per-GPU power signals +using the deterministic formulas from §5.2 of the design, applies bias × noise +from the scenario timeline, and exposes results via ``observation_at(tick)`` +(same interface as SyntheticFleet so FakePrometheusClient is class-agnostic). +""" + +from __future__ import annotations + +import random +from dataclasses import dataclass +from typing import TYPE_CHECKING, Any, Optional + +if TYPE_CHECKING: + from dynamo.planner.tests.testbed.fake_actuator import AppliedCaps + from dynamo.planner.tests.testbed.scenarios import ( + OverlaySpec, + ScenarioSpec, + SystemSpec, + ) + + +@dataclass +class OverlayObservation: + """Power signals synthesised from one tick's FPM snapshots.""" + + tick: int + power_w_prefill: float # per-GPU average + power_w_decode: float # per-GPU average + + # Compatibility with SyntheticFleet.observation_at() interface: + # FakePrometheusClient checks .ttft_avg_s, .itl_avg_s as well. + # We expose None for latency fields — FakePrometheus returns None for + # those signals from γ-class (latency comes from mocker traffic drain). + ttft_avg_s: Optional[float] = None + itl_avg_s: Optional[float] = None + total_tokens_per_sec: Optional[float] = None + traffic: Any = None + + +class SyntheticPowerOverlay: + """Derives per-component power signals from mocker FPM snapshots. + + Implements the same ``observation_at(tick)`` + ``is_signal_in_outage()`` + + ``prom_stale_lag()`` interface as SyntheticFleet so FakePrometheusClient + is a drop-in for both α and γ-class. + """ + + def __init__( + self, + overlay_spec: "OverlaySpec", + system_spec: "SystemSpec", + scenario: "ScenarioSpec", + rng: random.Random, + ) -> None: + self._spec = overlay_spec + self._system = system_spec + self._scenario = scenario + self._rng = rng + + # Current applied caps (updated by ReplayFakeActuator via notify_caps_changed) + self._applied_cap_p: int = scenario.planner.prefill_engine_gpu_power_limit + self._applied_cap_d: int = scenario.planner.decode_engine_gpu_power_limit + + self._history: dict[int, OverlayObservation] = {} + self._latest: Optional[OverlayObservation] = None + + # Active observability faults (copied from fleet state interface) + self._active_prom_outage: dict[str, int] = {} + self._active_prom_stale: Optional[tuple[int, int]] = None + self._ar1_state: dict[str, float] = {} + + # ------------------------------------------------------------------ + # Main per-tick observation + # ------------------------------------------------------------------ + + def observe( + self, + fpm_snapshots: list[dict[str, Any]], + applied_caps: "AppliedCaps", + tick: int, + ) -> None: + """Synthesise power signals from this tick's FPM snapshots. + + Called after bridge.advance_to() returns snapshots for the tick. + """ + prefill_snaps = [s for s in fpm_snapshots if s.get("component") == "prefill"] + decode_snaps = [s for s in fpm_snapshots if s.get("component") == "decode"] + + # Honour the overlay spec's own bias defaults — these are the + # "steady-state" multipliers the scenario author set on + # ``overlay.bias.power_bias_*``. Events (BiasStepEvent / BiasRampEvent) + # below can override the per-component bias dynamically per tick. + bias_p = self._spec.bias.power_bias_prefill + bias_d = self._spec.bias.power_bias_decode + + # Read scenario bias timeline for this tick + for event in self._scenario.parsed_events(): + from dynamo.planner.tests.testbed.scenarios import ( + BiasRampEvent, + BiasStepEvent, + ) + + if isinstance(event, BiasStepEvent) and event.at_tick <= tick: + if "prefill" in event.signal: + bias_p = event.value + elif "decode" in event.signal: + bias_d = event.value + elif isinstance(event, BiasRampEvent): + if event.start_tick <= tick <= event.end_tick: + t = (tick - event.start_tick) / max( + 1, event.end_tick - event.start_tick + ) + val = event.from_ + t * (event.to - event.from_) + if "prefill" in event.signal: + bias_p = val + elif "decode" in event.signal: + bias_d = val + + p_w = self._aggregate_power( + prefill_snaps, applied_caps.cap_p, self._predict_prefill_power + ) + d_w = self._aggregate_power( + decode_snaps, applied_caps.cap_d, self._predict_decode_power + ) + + p_w *= bias_p * (1.0 + self._noise("power_per_gpu")) + d_w *= bias_d * (1.0 + self._noise("power_per_gpu")) + + obs = OverlayObservation( + tick=tick, + power_w_prefill=max(0.0, p_w), + power_w_decode=max(0.0, d_w), + ) + self._history[tick] = obs + self._latest = obs + + def observation_at(self, tick: int) -> Optional[OverlayObservation]: + return self._history.get(tick) + + def notify_caps_changed(self, new_cap_p: int, new_cap_d: int) -> None: + """Called by ReplayFakeActuator when caps are applied (possibly clamped).""" + self._applied_cap_p = new_cap_p + self._applied_cap_d = new_cap_d + + # ------------------------------------------------------------------ + # FakePrometheusClient interface stubs + # ------------------------------------------------------------------ + + def is_signal_in_outage(self, signal: str) -> bool: + return signal in self._active_prom_outage + + def prom_stale_lag(self) -> Optional[int]: + if self._active_prom_stale: + return self._active_prom_stale[1] + return None + + def set_prom_outage(self, signals: list[str], end_tick: int) -> None: + for sig in signals: + self._active_prom_outage[sig] = end_tick + + def clear_expired_events(self, tick: int) -> None: + expired = [s for s, end in self._active_prom_outage.items() if tick >= end] + for s in expired: + del self._active_prom_outage[s] + if self._active_prom_stale and tick >= self._active_prom_stale[0]: + self._active_prom_stale = None + + # ------------------------------------------------------------------ + # Power prediction formulas (§5.2) + # ------------------------------------------------------------------ + + def _predict_prefill_power(self, snap: dict[str, Any], applied_cap_w: int) -> float: + """Compute-bound regime. Power scales with GEMM intensity, clamped by cap.""" + tdp = self._system.tdp_w + sku_min = self._system.sku_min_w + sat_tokens = self._system.overlay_prefill_saturation_tokens + base = 0.6 * tdp + gemm_load = min(1.0, snap.get("sum_prefill_tokens", 0) / max(1, sat_tokens)) + aic_predicted = base + (applied_cap_w - base) * gemm_load + return max(sku_min, min(applied_cap_w, aic_predicted)) + + def _predict_decode_power(self, snap: dict[str, Any], applied_cap_w: int) -> float: + """Memory-bound regime. Power scales with KV traffic, less sensitive to cap.""" + tdp = self._system.tdp_w + sku_min = self._system.sku_min_w + hbm_tokens = self._system.overlay_decode_hbm_tokens + base = 0.5 * tdp + hbm_load = min(1.0, snap.get("sum_decode_kv_tokens", 0) / max(1, hbm_tokens)) + aic_predicted = base + (applied_cap_w - base) * (0.3 + 0.7 * hbm_load) + return max(sku_min, min(applied_cap_w, aic_predicted)) + + def _aggregate_power( + self, + snaps: list[dict[str, Any]], + applied_cap_w: int, + predict_fn, + ) -> float: + if not snaps: + # Fall back to system TDP × idle fraction when no FPMs + return self._system.tdp_w * 0.5 + total = sum(predict_fn(s, applied_cap_w) for s in snaps) + return total / len(snaps) + + # ------------------------------------------------------------------ + # Noise + # ------------------------------------------------------------------ + + def _noise(self, signal: str) -> float: + spec = getattr(self._spec.noise, signal, None) or self._spec.noise.power_per_gpu + from dynamo.planner.tests.testbed.synthetic_fleet import _sample_noise + + return _sample_noise(spec, self._rng, f"{signal}_ar1", self._ar1_state) diff --git a/components/src/dynamo/planner/tests/testbed/runner.py b/components/src/dynamo/planner/tests/testbed/runner.py new file mode 100644 index 000000000000..5f8bfc27f86f --- /dev/null +++ b/components/src/dynamo/planner/tests/testbed/runner.py @@ -0,0 +1,633 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""ScenarioRunner — dispatches α and γ scenarios, runs the closed-loop tick. + +α-class per-tick data flow (§2.1 of testbed design): + 1. Apply scenario events for this tick + 2. Get offered load from load profile + 3. SyntheticFleet.step() → Observation + 4. FakePrometheus updates (via source) + 5. optimizer.update_correction(...) + 6. if optimizer.should_reoptimize(): optimize() → apply caps + 7. state_machine._apply_power_budget(desired_p, desired_d) → clamped + 8. FakeActuator.apply_replicas(...) + 9. TickRecorder.record(...) + +γ-class delegates to PowerAwareReplayAdapter.run() after setup. +""" + +from __future__ import annotations + +import argparse +import logging +import random +import time +from pathlib import Path +from typing import TYPE_CHECKING, Optional + +if TYPE_CHECKING: + from dynamo.planner.tests.testbed.recorder import TickHistory + from dynamo.planner.tests.testbed.scenarios import ScenarioSpec + +logger = logging.getLogger(__name__) + + +class ScenarioRunner: + """Main harness for a single scenario. + + Usage:: + + spec = load_scenario("scenarios/A1_power_under_estimate_decode.yaml") + runner = ScenarioRunner(spec) + history = runner.run() + """ + + def __init__(self, scenario: "ScenarioSpec") -> None: + self.scenario = scenario + + # ------------------------------------------------------------------ + # α-class setup + # ------------------------------------------------------------------ + + def _setup_alpha(self): + """Build all α-class fakes and inject seams into production code.""" + from dynamo.planner.config.aic_interpolation_spec import AICInterpolationSpec + from dynamo.planner.config.parallelization import PickedParallelConfig + from dynamo.planner.config.planner_config import PlannerConfig + from dynamo.planner.core.state_machine import PlannerStateMachine + from dynamo.planner.core.types import EngineCapabilities, WorkerCapabilities + from dynamo.planner.monitoring.aic_power_optimizer import AICPowerOptimizer + from dynamo.planner.tests.testbed.clock import Clock + from dynamo.planner.tests.testbed.fake_actuator import FakeActuator + from dynamo.planner.tests.testbed.fake_aic import FakeAIC + from dynamo.planner.tests.testbed.fake_planner_metrics import FakePlannerMetrics + from dynamo.planner.tests.testbed.fake_prometheus import FakePrometheusClient + from dynamo.planner.tests.testbed.recorder import TickHistory + from dynamo.planner.tests.testbed.scenarios import SystemSpec + from dynamo.planner.tests.testbed.synthetic_fleet import SyntheticFleet + + sc = self.scenario + rng = random.Random(sc.seed) + + system_spec = SystemSpec.load(sc.fleet.system) + + self.fleet = SyntheticFleet(sc.fleet, system_spec, sc, rng) + self.metrics = FakePlannerMetrics() + self.actuator = FakeActuator(sc, self.fleet, self.metrics, system_spec) + self.prom = FakePrometheusClient(source=self.fleet) + self.fake_aic = FakeAIC(system_spec) + self.clock = Clock(interval_s=sc.interval_s) + + # Build PlannerConfig from scenario planner spec + planner_spec = sc.planner + aic_spec = AICInterpolationSpec( + hf_id="fake-model/testbed", + system=sc.fleet.system, + backend="vllm", + isl=3000, + osl=150, + sweep_max_context_length=8192, + prefill_interpolation_granularity=1, + decode_interpolation_granularity=1, + prefill_pick=PickedParallelConfig(tp=1, pp=1, dp=1), + decode_pick=PickedParallelConfig(tp=1, pp=1, dp=1), + ) + self.config = PlannerConfig( + mode=planner_spec.mode, + ttft=planner_spec.ttft, + itl=planner_spec.itl, + enable_power_awareness=planner_spec.enable_power_awareness, + enable_aic_optimizer=planner_spec.enable_aic_optimizer, + total_gpu_power_limit=planner_spec.total_gpu_power_limit, + power_agent_safe_default_watts=planner_spec.power_agent_safe_default_watts, + prefill_engine_gpu_power_limit=planner_spec.prefill_engine_gpu_power_limit, + decode_engine_gpu_power_limit=planner_spec.decode_engine_gpu_power_limit, + aic_initial_c_power_prefill=planner_spec.aic_initial_c_power_prefill, + aic_initial_c_power_decode=planner_spec.aic_initial_c_power_decode, + aic_initial_c_power_agg=planner_spec.aic_initial_c_power_agg, + aic_initial_c_ttft=planner_spec.aic_initial_c_ttft, + aic_initial_c_itl=planner_spec.aic_initial_c_itl, + aic_reoptimize_interval=planner_spec.aic_reoptimize_interval, + aic_drift_relative_threshold=planner_spec.aic_drift_relative_threshold, + aic_drift_consecutive_ticks=planner_spec.aic_drift_consecutive_ticks, + aic_max_consecutive_failures=planner_spec.aic_max_consecutive_failures, + min_endpoint=planner_spec.min_endpoint, + max_gpu_budget=planner_spec.max_gpu_budget, + aic_interpolation=aic_spec, + # disable report generation in testbed + live_dashboard_port=0, + report_interval_hours=None, + ) + + # Build WorkerCapabilities for state machine + caps = WorkerCapabilities( + prefill=EngineCapabilities(num_gpu=sc.fleet.gpus_per_prefill_engine), + decode=EngineCapabilities(num_gpu=sc.fleet.gpus_per_decode_engine), + ) + self.state_machine = PlannerStateMachine(self.config, caps) + + # Parse events once at setup — Pydantic model construction is the + # bottleneck for long scenarios (200 ticks × N events × 2 reparse). + self._events = sc.parsed_events() + + # ------------------------------------------------------------------ + # Step 1 — process environmental events scheduled at tick 0 BEFORE + # the cold-start sweep. These represent cluster state present at + # planner startup (e.g. an NVML clamp fault is already in effect when + # the planner boots). Without this, ``apply_caps()`` in setup never + # sees the fault and B7/B8-style clamp scenarios silently no-op. + # We re-fire these in ``_tick_alpha`` (set_actuation_fault etc. are + # idempotent) so existing event-driven assertions still see them. + # ------------------------------------------------------------------ + from dynamo.planner.tests.testbed.scenarios import ( + ActuationFaultEvent, + AicFailureEvent, + BudgetChangeEvent, + FrontendPostFaultEvent, + ) + + for event in self._events: + if getattr(event, "at_tick", None) != 0: + continue + if isinstance(event, ActuationFaultEvent): + self.actuator.set_actuation_fault(event.mode) + elif isinstance(event, AicFailureEvent): + self.fake_aic.set_fault_mode(event.mode) + self._aic_fault_reset_tick = event.n_consecutive + elif isinstance(event, BudgetChangeEvent): + self.config.total_gpu_power_limit = event.new_total_w + elif isinstance(event, FrontendPostFaultEvent): + # Install the frontend POST fault on the fleet so the + # cold-start /busy_threshold fan-out can hit it. + self.fleet.apply_event(event, 0) + + # ------------------------------------------------------------------ + # Step 2 — apply the *configured* per-engine caps first. Production + # planner does this via ``_apply_power_annotations`` on the first + # tick (before the AIC optimizer's first sweep returns). Doing it + # here lets B7/B8 actually exercise NVML clamping when the + # configured value lies outside [sku_min, sku_max]. + # ------------------------------------------------------------------ + self.actuator.apply_caps( + int(planner_spec.prefill_engine_gpu_power_limit), + int(planner_spec.decode_engine_gpu_power_limit), + ) + + if planner_spec.enable_aic_optimizer: + # Patch time.monotonic with the virtual clock BEFORE constructing + # the optimizer so its initial _time_of_last_optimize reads 0.0 in + # virtual time, not wall-clock time. + import unittest.mock as mock + + self._clock_patch = mock.patch("time.monotonic", side_effect=self.clock.now) + self._clock_patch.start() + + self.optimizer = AICPowerOptimizer(self.config, self.metrics) + self.optimizer._aic_estimator_factory = ( + self.fake_aic.make_estimator_factory() + ) + + # Run the cold-start sweep — mirrors production base.py setup_async() + # and γ-adapter PowerAwareReplayAdapter.__init__. Without this, + # _last_optimal_config stays None and update_correction() is a + # permanent no-op (every α scenario silently fails to drive c_*). + initial = self.optimizer.optimize() + if initial is not None: + self.config.prefill_engine_gpu_power_limit = initial.cap_p + self.config.decode_engine_gpu_power_limit = initial.cap_d + self.actuator.apply_caps(initial.cap_p, initial.cap_d) + # Seed truth-side replica counts AFTER the power-budget clamp + # so subsequent ticks start at a feasible point. Using the + # raw optimizer pick (initial.n_p/n_d) overruns budget — the + # state machine clamps to (final_p, final_d). Apply that + # clamp now so the truth state matches production behavior. + final_p, final_d = self.state_machine._apply_power_budget( + initial.n_p, initial.n_d + ) + # Set ``_estimated_throughput`` from the **post-budget-clamp** + # replica count, not the optimizer's raw pick. Production has + # the same shape but the discrepancy is hidden by other layers; + # here, the synthetic fleet's truth capacity is tied directly + # to ``n_d_truth``, so without this fix the drift detector + # compares observed traffic against a fantasy capacity and + # the capacity_exceeded trigger never fires (F26, C13). + self.optimizer._estimated_throughput = ( + initial.aic_seq_per_s_per_replica + * final_d + * (initial.isl + initial.osl) + ) + self.fleet.state.n_p_truth = final_p + self.fleet.state.n_d_truth = final_d + self.actuator.apply_replicas(final_p, final_d) + if sc.planner.admission_mode == "autoset": + self._fanout_busy_threshold_posts() + else: + self.optimizer = None + self._clock_patch = None + + self.history = TickHistory() + + # ------------------------------------------------------------------ + # α-class tick loop + # ------------------------------------------------------------------ + + def _tick_alpha(self, tick: int) -> None: + from dynamo.planner.tests.testbed.recorder import TickSnapshot + from dynamo.planner.tests.testbed.scenarios import ( + ActuationFaultEvent, + AicFailureEvent, + BudgetChangeEvent, + ) + + sc = self.scenario + self.clock.advance(tick) + self.prom.set_tick(tick) + + # Apply events scheduled at this tick + for event in self._events: + event_at = getattr(event, "at_tick", None) + if event_at != tick: + continue + if isinstance(event, AicFailureEvent): + if self.fake_aic: + self.fake_aic.set_fault_mode(event.mode) + self._aic_fault_reset_tick = tick + event.n_consecutive + elif isinstance(event, BudgetChangeEvent): + self.config.total_gpu_power_limit = event.new_total_w + elif isinstance(event, ActuationFaultEvent): + self.actuator.set_actuation_fault(event.mode) + else: + self.fleet.apply_event(event, tick) + + # Clear expired AIC fault + if ( + hasattr(self, "_aic_fault_reset_tick") + and tick >= self._aic_fault_reset_tick + ): + if self.fake_aic: + self.fake_aic.reset_fault() + + # Clear expired fleet events + self.fleet.clear_expired_events(tick) + + # Clear expired actuation fault + for event in self._events: + if not isinstance(event, ActuationFaultEvent): + continue + if event.at_tick + event.duration_ticks <= tick: + self.actuator.set_actuation_fault(None) + + # Step the fleet + offered_load = sc.offered_load_at(tick) + obs = self.fleet.step(tick, offered_load) + + # Capture only pre-tick pegged baselines (we want per-tick *event* + # semantics for ``correction_pegged`` — "did the coefficient clamp + # *this tick*"). Other counter snapshot fields below use cumulative + # values (natural Prometheus counter semantics) so assertions like + # ``cap_clamped_min > 0 at tick 10`` read "has clamping happened by + # tick 10", which matches the scenarios' intent. + before_pegged = dict(self._read_pegged_counters()) + + # AIC optimizer update + potential re-sweep + sweep_fired = False + # Default desired = current truth state; sweep can override below. + desired_p = self.fleet.state.n_p_truth + desired_d = self.fleet.state.n_d_truth + + if self.optimizer is not None and obs.traffic.num_req is not None: + self.optimizer.update_correction( + traffic=obs.traffic, + observed_ttft_avg=obs.ttft_avg_s, + observed_itl_avg=obs.itl_avg_s, + observed_power_w_prefill=obs.power_w_prefill, + observed_power_w_decode=obs.power_w_decode, + ) + # When admission_mode == "autoset", production's + # ``_apply_aic_config`` fans out a /busy_threshold POST to every + # frontend pod after every sweep AND on cold-start, then the + # planner relies on those pods enforcing the threshold. The + # testbed mirrors that fan-out (3 synthetic pods) only on + # sweeps so the B11 frontend-POST-fault counter is reachable. + if self.optimizer.should_reoptimize(obs.traffic): + new_cfg = self.optimizer.optimize() + if new_cfg is not None: + self.config.prefill_engine_gpu_power_limit = new_cfg.cap_p + self.config.decode_engine_gpu_power_limit = new_cfg.cap_d + self.actuator.apply_caps(new_cfg.cap_p, new_cfg.cap_d) + # Honor the optimizer's replica recommendation — this is + # the path that exercises _apply_power_budget's clamp + + # min_endpoint enforcement (scenarios A6, E22, E25). + desired_p = new_cfg.n_p + desired_d = new_cfg.n_d + # Estimated throughput uses the *post-budget-clamp* n_d + # so drift detection compares against achievable capacity. + # See note in ``_setup_alpha`` for the production-divergence + # rationale (testbed scenarios assume post-clamp semantics). + ( + sweep_final_p, + sweep_final_d, + ) = self.state_machine._apply_power_budget(new_cfg.n_p, new_cfg.n_d) + self.optimizer._estimated_throughput = ( + new_cfg.aic_seq_per_s_per_replica + * sweep_final_d + * (new_cfg.isl + new_cfg.osl) + ) + sweep_fired = True + + # Fan out /busy_threshold POSTs for autoset admission. + # Pure synthetic — no real network — but exercises the + # frontend-POST fault path (B11) and respects asyncio + # boundary by gathering results. + if sc.planner.admission_mode == "autoset": + self._fanout_busy_threshold_posts() + + # Power budget clamp + final_p, final_d = self.state_machine._apply_power_budget(desired_p, desired_d) + + # Apply replicas (may fault) + try: + self.actuator.apply_replicas(final_p, final_d) + except RuntimeError: + pass # RBAC fault; state unchanged + + # Compute projected power + cap_p = self.fleet.state.applied_cap_p + cap_d = self.fleet.state.applied_cap_d + gpus_p = sc.fleet.gpus_per_prefill_engine + gpus_d = sc.fleet.gpus_per_decode_engine + projected_w = ( + self.fleet.state.n_p_truth * cap_p * gpus_p + + self.fleet.state.n_d_truth * cap_d * gpus_d + ) + + # Read optimizer state + c_ttft = c_itl = c_power_p = c_power_d = 1.0 + estimated_throughput = 0.0 + consecutive_violations = 0 + if self.optimizer: + c_ttft = self.optimizer._c_ttft + c_itl = self.optimizer._c_itl + c_power_p = self.optimizer._c_power_p + c_power_d = self.optimizer._c_power_d + estimated_throughput = self.optimizer._estimated_throughput + consecutive_violations = self.optimizer._consecutive_violation_ticks + + after_pegged = self._read_pegged_counters() + pegged_delta = { + k: after_pegged.get(k, 0) - before_pegged.get(k, 0) for k in after_pegged + } + + n_oscillations = self._update_oscillation_count( + self.fleet.state.n_p_truth, self.fleet.state.n_d_truth + ) + + snap = TickSnapshot( + tick=tick, + n_p=self.fleet.state.n_p_truth, + n_d=self.fleet.state.n_d_truth, + cap_p=self.fleet.state.applied_cap_p, + cap_d=self.fleet.state.applied_cap_d, + observed_ttft_s=obs.ttft_avg_s, + observed_itl_s=obs.itl_avg_s, + observed_power_w_p=obs.power_w_prefill, + observed_power_w_d=obs.power_w_decode, + observed_capacity_tps=obs.total_tokens_per_sec, + c_ttft=c_ttft, + c_itl=c_itl, + c_power_p=c_power_p, + c_power_d=c_power_d, + estimated_throughput=estimated_throughput, + consecutive_violation_ticks=consecutive_violations, + projected_w=projected_w, + budget_w=float(self.config.total_gpu_power_limit or 0), + sweep_fired=sweep_fired, + sla_violated=( + obs.ttft_avg_s > (sc.planner.ttft / 1000.0) + or obs.itl_avg_s > (sc.planner.itl / 1000.0) + ), + capacity_exceeded=obs.total_tokens_per_sec > (estimated_throughput * 1.15) + if estimated_throughput > 0 + else False, + cap_clamped_min=int( + self.metrics.power_agent_cap_clamped_total.labeled_value( + direction="min" + ) + ), + cap_clamped_max=int( + self.metrics.power_agent_cap_clamped_total.labeled_value( + direction="max" + ) + ), + optimizer_exceptions=int(self.metrics.aic_optimizer_exceptions_total.value), + correction_pegged={k: v for k, v in pegged_delta.items() if v > 0}, + admission_partial_failures=int( + self.metrics.admission_partial_success_total.value + ), + n_oscillations=n_oscillations, + ) + self.history.append(snap) + + # Synthesised count of frontend pods receiving /busy_threshold POSTs. + # 3 is enough to make the per-call Bernoulli fault model converge to + # the configured failing_fraction over a 10-tick window. + _FRONTEND_POD_COUNT = 3 + + def _fanout_busy_threshold_posts(self) -> None: + """Synthetic equivalent of ``base.py::_apply_aic_config``'s POST fan-out. + + Calls ``actuator.post_busy_threshold`` for each synthetic frontend + pod. Exceptions (synthetic 503s under FrontendPostFaultEvent) are + swallowed here — they're already accounted for in the + admission_partial_success_total counter inside the actuator. + """ + import asyncio + + async def _gather(): + await asyncio.gather( + *( + self.actuator.post_busy_threshold( + pod=f"frontend-{i}", + model="fake-model/testbed", + port=8080, + active_decode_blocks_threshold=0.97, + active_prefill_tokens_threshold=4096, + active_prefill_tokens_threshold_frac=1.0, + ) + for i in range(self._FRONTEND_POD_COUNT) + ), + return_exceptions=True, + ) + + try: + asyncio.run(_gather()) + except RuntimeError: + # Caller already running event loop (γ adapter); fall through + # via run-until-complete on the loop. + loop = asyncio.new_event_loop() + try: + loop.run_until_complete(_gather()) + finally: + loop.close() + + def _update_oscillation_count(self, n_p: int, n_d: int) -> int: + """Maintain a cumulative replica direction-flip counter. + + Increments when the sign of (n_p_now − n_p_prev) flips relative to the + previous non-zero delta — same for n_d. Used by G2's "at most 3 + flip-flops" assertion. + """ + prev_snap = self.history.snapshots[-1] if self.history.snapshots else None + if prev_snap is None: + self._osc_count = 0 + self._osc_last_dp = 0 + self._osc_last_dd = 0 + self._osc_prev_n_p = n_p + self._osc_prev_n_d = n_d + return 0 + + dp = n_p - self._osc_prev_n_p + dd = n_d - self._osc_prev_n_d + + if dp != 0: + if self._osc_last_dp != 0 and (dp > 0) != (self._osc_last_dp > 0): + self._osc_count += 1 + self._osc_last_dp = dp + if dd != 0: + if self._osc_last_dd != 0 and (dd > 0) != (self._osc_last_dd > 0): + self._osc_count += 1 + self._osc_last_dd = dd + + self._osc_prev_n_p = n_p + self._osc_prev_n_d = n_d + return self._osc_count + + def _read_pegged_counters(self) -> dict[str, float]: + c = self.metrics.aic_correction_pegged_total + result = {} + for key, labeled in c._labels.items(): + coeff = dict(key).get("coefficient", "unknown") + result[coeff] = labeled.value + return result + + # ------------------------------------------------------------------ + # γ-class setup + run + # ------------------------------------------------------------------ + + def _setup_and_run_gamma(self) -> "TickHistory": + from dynamo.planner.tests.testbed.replay.power_aware_replay_adapter import ( + build_gamma_adapter, + ) + + return build_gamma_adapter(self.scenario).run_and_record() + + # ------------------------------------------------------------------ + # Public + # ------------------------------------------------------------------ + + def run( + self, + csv_path: Optional[Path] = None, + prom_path: Optional[Path] = None, + plot_path: Optional[Path] = None, + ) -> "TickHistory": + sc = self.scenario + t0 = time.perf_counter() + + if sc.class_name == "alpha": + self._setup_alpha() + for tick in range(sc.ticks): + self._tick_alpha(tick) + if self._clock_patch: + self._clock_patch.stop() + history = self.history + else: + history = self._setup_and_run_gamma() + + elapsed = time.perf_counter() - t0 + logger.info( + "Scenario %s completed %d ticks in %.2fs", sc.name, len(history), elapsed + ) + + if csv_path: + history.to_csv(Path(csv_path)) + if prom_path: + history.to_prom_textfile(Path(prom_path), sc.name) + if plot_path: + history.plot(Path(plot_path), sc.name) + + return history + + +# --------------------------------------------------------------------------- +# CLI entrypoint +# --------------------------------------------------------------------------- + + +def main() -> None: + parser = argparse.ArgumentParser( + description="Power Planner stress testbed runner", + ) + parser.add_argument("--scenario", help="Scenario name (e.g. A1) or full YAML path") + parser.add_argument("--all", action="store_true", help="Run all scenarios") + parser.add_argument( + "--class-filter", choices=["alpha", "gamma", "all"], default="all" + ) + parser.add_argument("--csv", help="Output CSV path (single scenario)") + parser.add_argument("--csv-dir", help="Output CSV directory (--all mode)") + parser.add_argument("--plot", help="Output PNG path (single scenario)") + parser.add_argument("--prom-textfile", help="Prometheus textfile output path") + parser.add_argument("-v", "--verbose", action="store_true") + args = parser.parse_args() + + logging.basicConfig(level=logging.DEBUG if args.verbose else logging.INFO) + + from dynamo.planner.tests.testbed.scenarios import load_all_scenarios, load_scenario + + scenarios_dir = Path(__file__).parent / "scenarios" + + if args.all: + results = [] + all_specs = load_all_scenarios() + for name, spec in all_specs: + if args.class_filter != "all" and spec.class_name != args.class_filter: + continue + csv_path = Path(args.csv_dir) / f"{name}.csv" if args.csv_dir else None + runner = ScenarioRunner(spec) + try: + runner.run(csv_path=csv_path) + results.append((name, "PASS")) + except Exception as e: + results.append((name, f"FAIL: {e}")) + for name, status in results: + print(f" {status:8s} {name}") + failed = [r for r in results if not r[1].startswith("PASS")] + if failed: + raise SystemExit(f"{len(failed)} scenario(s) FAILED") + else: + if not args.scenario: + parser.error("Either --scenario or --all is required") + # Resolve scenario + if Path(args.scenario).exists(): + path = Path(args.scenario) + else: + # Search by prefix + candidates = list(scenarios_dir.glob(f"{args.scenario}*.yaml")) + if not candidates: + raise SystemExit( + f"No scenario matching {args.scenario!r} in {scenarios_dir}" + ) + path = candidates[0] + spec = load_scenario(path) + runner = ScenarioRunner(spec) + runner.run( + csv_path=args.csv, + prom_path=args.prom_textfile, + plot_path=args.plot, + ) + print(f"PASS {spec.name}") + + +if __name__ == "__main__": + main() diff --git a/components/src/dynamo/planner/tests/testbed/scenarios.py b/components/src/dynamo/planner/tests/testbed/scenarios.py new file mode 100644 index 000000000000..e3f890773858 --- /dev/null +++ b/components/src/dynamo/planner/tests/testbed/scenarios.py @@ -0,0 +1,604 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Scenario Pydantic models + YAML loader with ``extends:`` inheritance. + +Schema summary: + - ScenarioSpec (top-level) — class, seed, ticks, interval_s, planner, fleet|mocker+overlay, load, events, assertions + - PlannerSpec — mirrors key PlannerConfig fields for testbed + - FleetSpec (α-class) — system, gpus_per_*, decode_power_floor_w, bias, noise + - MockerSpec (γ-class) — trace_file, workload params, engine args + - OverlaySpec (γ-class) — system, noise + - LoadSpec — profile (constant|ramp|spike), tokens_per_sec + - Event — union of all event types (bias_step, actuation_fault, node_down, prom_outage, …) + - Assertion — structured or expression form + +YAML loader performs: + 1. Single-level ``extends:`` merge (scalar override, dict recursive merge, list replace). + 2. Pydantic validation on merged dict. + 3. Load-time assertion field validation against TickSnapshot field set. +""" + +from __future__ import annotations + +import ast +import math +from pathlib import Path +from typing import Annotated, Any, Literal, Optional, Union + +import yaml +from pydantic import BaseModel, Field, model_validator + +# --------------------------------------------------------------------------- +# System spec (loaded from systems/.yaml) +# --------------------------------------------------------------------------- + +_SYSTEMS_DIR = Path(__file__).parent / "systems" + + +class SystemSpec(BaseModel): + """Per-SKU hardware constants used by SyntheticFleet and FakeAICEstimator.""" + + tdp_w: float + sku_min_w: float + sku_max_w: float + decode_power_floor_w: float + # FakeAIC estimator constants + aic_ttft_ms: float + aic_itl_ms: float + aic_power_w_prefill: float + aic_power_w_decode: float + max_kv_tokens: int + # Overlay model constants (γ-class) + overlay_prefill_saturation_tokens: int = 8192 + overlay_decode_hbm_tokens: int = 200_000 + + @classmethod + def load(cls, system_name: str) -> "SystemSpec": + path = _SYSTEMS_DIR / f"{system_name}.yaml" + if not path.exists(): + raise FileNotFoundError(f"System spec not found: {path}") + with path.open() as f: + data = yaml.safe_load(f) + return cls(**data) + + +# --------------------------------------------------------------------------- +# Sub-specs +# --------------------------------------------------------------------------- + + +class NoiseModel(BaseModel): + model: Literal["gaussian", "uniform", "ar1"] = "gaussian" + sigma: float = 0.0 + half_width: float = 0.0 # uniform + rho: float = 0.9 # ar1 + # prev noise state is managed by SyntheticFleet, not stored here + + +class NoiseSpec(BaseModel): + power_per_gpu: NoiseModel = Field( + default_factory=lambda: NoiseModel(model="gaussian", sigma=0.07) + ) + ttft: NoiseModel = Field( + default_factory=lambda: NoiseModel(model="gaussian", sigma=0.05) + ) + itl: NoiseModel = Field( + default_factory=lambda: NoiseModel(model="gaussian", sigma=0.04) + ) + capacity: NoiseModel = Field( + default_factory=lambda: NoiseModel(model="gaussian", sigma=0.03) + ) + + +class BiasSpec(BaseModel): + power_bias_prefill: float = 1.0 + power_bias_decode: float = 1.0 + ttft_bias: float = 1.0 + itl_bias: float = 1.0 + capacity_bias: float = 1.0 + + +class FleetSpec(BaseModel): + system: str = "h200_sxm" + gpus_per_prefill_engine: int = 1 + gpus_per_decode_engine: int = 2 + decode_power_floor_w: Optional[float] = None # overrides system spec if set + bias: BiasSpec = Field(default_factory=BiasSpec) + noise: NoiseSpec = Field(default_factory=NoiseSpec) + + +class MockerSpec(BaseModel): + trace_file: Optional[str] = None + synthetic_workload: bool = False + arrival_rate: float = 200.0 + isl: int = 3000 + osl: int = 150 + trace_block_size: int = 512 + arrival_speedup_ratio: float = 1.0 + num_prefill_workers: int = 1 + num_decode_workers: int = 4 + router_mode: str = "kv_router" + prefill_engine_args: dict[str, Any] = Field(default_factory=dict) + decode_engine_args: dict[str, Any] = Field(default_factory=dict) + + +class OverlaySpec(BaseModel): + system: str = "h200_sxm" + bias: BiasSpec = Field(default_factory=BiasSpec) + noise: NoiseSpec = Field(default_factory=NoiseSpec) + + +class PlannerSpec(BaseModel): + mode: Literal["disagg", "agg"] = "disagg" + ttft: float = 500.0 # ms + itl: float = 50.0 # ms + enable_power_awareness: bool = True + enable_aic_optimizer: bool = True + total_gpu_power_limit: Optional[int] = 4000 + power_agent_safe_default_watts: int = 500 + prefill_engine_gpu_power_limit: int = 500 + decode_engine_gpu_power_limit: int = 425 + aic_initial_c_power_prefill: float = 1.0 + aic_initial_c_power_decode: float = 1.0 + aic_initial_c_power_agg: float = 1.0 + aic_initial_c_ttft: float = 1.0 + aic_initial_c_itl: float = 1.0 + aic_reoptimize_interval: int = 300 # seconds (virtual); 5 ticks at 60s + aic_drift_relative_threshold: float = 0.15 + aic_drift_consecutive_ticks: int = 3 + aic_max_consecutive_failures: int = 5 + min_endpoint: int = 1 + max_gpu_budget: int = 64 + admission_mode: Literal["off", "inherit", "autoset"] = "off" + + +class LoadSpec(BaseModel): + profile: Literal["constant", "ramp", "spike", "sine"] = "constant" + tokens_per_sec: float = 2000.0 + # ramp: start -> end over ramp_start_tick to ramp_end_tick + ramp_start_tick: int = 0 + ramp_end_tick: int = 50 + ramp_from: float = 200.0 + ramp_to: float = 2000.0 + # spike: spike_tick, spike_duration_ticks, spike_tokens_per_sec + spike_tick: int = 50 + spike_duration_ticks: int = 10 + spike_tokens_per_sec: float = 5000.0 + # sine: amplitude, period_ticks, offset_tps + sine_amplitude: float = 500.0 + sine_period_ticks: int = 40 + sine_offset_tps: float = 2000.0 + + +# --------------------------------------------------------------------------- +# Events +# --------------------------------------------------------------------------- + + +class BiasStepEvent(BaseModel): + type: Literal["bias_step"] + at_tick: int + signal: str + value: float + auto_inject_window_cross: bool = False + + +class BiasRampEvent(BaseModel): + type: Literal["bias_ramp"] + start_tick: int + end_tick: int + signal: str + from_: float = Field(alias="from") + to: float + + model_config = {"populate_by_name": True} + + +class BiasSineEvent(BaseModel): + type: Literal["bias_sine"] + signal: str + amplitude: float + period_ticks: int + offset: float = 0.0 + + +class ActuationFaultEvent(BaseModel): + type: Literal["actuation_fault"] + at_tick: int + duration_ticks: int + mode: Literal["rbac_denied", "nvml_low", "nvml_high", "daemonset_absent"] + auto_inject_window_cross: bool = False + + +class NodeDownEvent(BaseModel): + type: Literal["node_down"] + at_tick: int + n_prefill_lost: int = 0 + n_decode_lost: int = 0 + + +class NodeUpEvent(BaseModel): + type: Literal["node_up"] + at_tick: int + n_prefill_restored: int = 0 + n_decode_restored: int = 0 + + +class PromOutageEvent(BaseModel): + type: Literal["prom_outage"] + at_tick: int + duration_ticks: int + signals: list[str] = Field( + default_factory=lambda: ["ttft", "itl", "power_p", "power_d", "capacity"] + ) + + +class PromStaleEvent(BaseModel): + type: Literal["prom_stale"] + at_tick: int + duration_ticks: int + lag_ticks: int = 5 + + +class PromWindowCrossEvent(BaseModel): + type: Literal["prom_window_cross_event"] + at_tick: int + signal: str + weight_old: float = 0.66 + + +class BudgetChangeEvent(BaseModel): + type: Literal["budget_change"] + at_tick: int + new_total_w: int + + +class FrontendPostFaultEvent(BaseModel): + type: Literal["frontend_post_fault"] + at_tick: int + duration_ticks: int + failing_fraction: float = 0.33 + + +class MdcUnavailableEvent(BaseModel): + type: Literal["mdc_unavailable"] + at_tick: int + duration_ticks: int + + +class AicFailureEvent(BaseModel): + type: Literal["aic_failure"] + at_tick: int + mode: Literal["empty_pareto", "raises"] + n_consecutive: int = 1 + + +Event = Annotated[ + Union[ + BiasStepEvent, + BiasRampEvent, + BiasSineEvent, + ActuationFaultEvent, + NodeDownEvent, + NodeUpEvent, + PromOutageEvent, + PromStaleEvent, + PromWindowCrossEvent, + BudgetChangeEvent, + FrontendPostFaultEvent, + MdcUnavailableEvent, + AicFailureEvent, + ], + Field(discriminator="type"), +] + + +# --------------------------------------------------------------------------- +# Assertions +# --------------------------------------------------------------------------- + + +# Valid assertion ``field:`` names — derived from TickSnapshot directly to +# guarantee the two stay in sync as fields are added/renamed. +def _load_tick_snapshot_fields() -> frozenset[str]: + from dynamo.planner.tests.testbed.recorder import TICK_SNAPSHOT_FIELDS + + return frozenset(TICK_SNAPSHOT_FIELDS) + + +# Cached at first access; recorder is a sibling module so the cycle is fine. +_TICK_SNAPSHOT_FIELDS: Optional[frozenset[str]] = None + + +def _tick_snapshot_fields() -> frozenset[str]: + global _TICK_SNAPSHOT_FIELDS + if _TICK_SNAPSHOT_FIELDS is None: + _TICK_SNAPSHOT_FIELDS = _load_tick_snapshot_fields() + return _TICK_SNAPSHOT_FIELDS + + +_ASSERTION_OPS = {"<", "<=", "==", ">=", ">", "within", "!="} + + +class StructuredAssertion(BaseModel): + field: Optional[str] = None + op: Optional[str] = None + value: Optional[float] = None + tolerance: Optional[float] = None + ref: Optional[str] = None + description: Optional[str] = None + # Exactly one of: at_tick (int), always (bool=True), eventually_by_tick (int). + at_tick: Optional[int] = None + always: Optional[bool] = None + eventually_by_tick: Optional[int] = None + # Counter form + counter: Optional[str] = None # counter name + label: Optional[dict[str, str]] = None + + @model_validator(mode="after") + def _validate_assertion(self) -> "StructuredAssertion": + if self.op is not None and self.op not in _ASSERTION_OPS: + raise ValueError( + f"Unknown op: {self.op!r}. Must be one of {_ASSERTION_OPS}" + ) + if self.op == "within" and self.tolerance is None: + raise ValueError("op='within' requires tolerance") + + # Exactly one predicate must be set, and `always` must be True if + # present (bare ``always:`` in YAML parses to None which would + # silently skip evaluation — that bit us hard once, never again). + predicates = [ + self.at_tick is not None, + self.always is True, + self.eventually_by_tick is not None, + ] + if sum(predicates) == 0: + raise ValueError( + "StructuredAssertion requires exactly one of " + "{at_tick: , always: true, eventually_by_tick: }. " + "Bare `always:` in YAML parses to None and is rejected — " + "write `always: true` explicitly." + ) + if sum(predicates) > 1: + raise ValueError( + "StructuredAssertion: at_tick / always / eventually_by_tick are " + "mutually exclusive — set exactly one." + ) + + # `field` / `op` are required for non-counter assertions. + if self.counter is None and (self.field is None or self.op is None): + raise ValueError( + "StructuredAssertion requires `field` and `op` " + "(or `counter` + `label` for counter-delta form)." + ) + return self + + +class ExprAssertion(BaseModel): + expr: str + at_tick: Optional[int] = None + always: Optional[bool] = None + eventually_by_tick: Optional[int] = None + description: Optional[str] = None + + +Assertion = Union[StructuredAssertion, ExprAssertion] + + +# --------------------------------------------------------------------------- +# Top-level scenario spec +# --------------------------------------------------------------------------- + + +class ScenarioSpec(BaseModel): + name: str + class_: Literal["alpha", "gamma"] = Field(alias="class", default="alpha") + description: str = "" + seed: int = 42 + ticks: int = 200 + interval_s: float = 60.0 + + planner: PlannerSpec = Field(default_factory=PlannerSpec) + fleet: Optional[FleetSpec] = None # α-class + mocker: Optional[MockerSpec] = None # γ-class + overlay: Optional[OverlaySpec] = None # γ-class + load: LoadSpec = Field(default_factory=LoadSpec) + events: list[dict[str, Any]] = Field(default_factory=list) + assertions: list[dict[str, Any]] = Field(default_factory=list) + + model_config = {"populate_by_name": True} + + @model_validator(mode="after") + def _validate_class_fields(self) -> "ScenarioSpec": + if self.class_ == "alpha" and self.fleet is None: + self.fleet = FleetSpec() + if self.class_ == "gamma" and self.mocker is None: + raise ValueError("gamma-class scenario requires a 'mocker:' block") + if self.class_ == "gamma" and self.overlay is None: + self.overlay = OverlaySpec() + return self + + @model_validator(mode="after") + def _validate_events_and_assertions(self) -> "ScenarioSpec": + """Eagerly parse events/assertions so authoring errors surface at load. + + Without this, a typo in a YAML event ``type:`` only blows up when the + runner actually consumes ``parsed_events()`` — which can be deep + inside a tick loop, far from a useful stack trace. + """ + # parsed_*() raises ValidationError on malformed entries. + self.parsed_events() + self.parsed_assertions() + # Reference-name validation against TickSnapshot field set. + errors = validate_assertion_fields(self.assertions) + if errors: + raise ValueError( + "Scenario assertion validation failed:\n " + "\n ".join(errors) + ) + return self + + @property + def class_name(self) -> str: + return self.class_ + + def offered_load_at(self, tick: int) -> float: + """Compute offered load (tok/s) for this tick according to load profile.""" + L = self.load + if L.profile == "constant": + return L.tokens_per_sec + elif L.profile == "ramp": + if tick <= L.ramp_start_tick: + return L.ramp_from + elif tick >= L.ramp_end_tick: + return L.ramp_to + t = (tick - L.ramp_start_tick) / max(1, L.ramp_end_tick - L.ramp_start_tick) + return L.ramp_from + t * (L.ramp_to - L.ramp_from) + elif L.profile == "spike": + if L.spike_tick <= tick < L.spike_tick + L.spike_duration_ticks: + return L.spike_tokens_per_sec + return L.tokens_per_sec + elif L.profile == "sine": + return L.sine_offset_tps + L.sine_amplitude * math.sin( + 2 * math.pi * tick / max(1, L.sine_period_ticks) + ) + return L.tokens_per_sec + + def parsed_events(self) -> list[Event]: + """Parse raw event dicts into typed Event objects.""" + result = [] + for e in self.events: + etype = e.get("type") + type_map = { + "bias_step": BiasStepEvent, + "bias_ramp": BiasRampEvent, + "bias_sine": BiasSineEvent, + "actuation_fault": ActuationFaultEvent, + "node_down": NodeDownEvent, + "node_up": NodeUpEvent, + "prom_outage": PromOutageEvent, + "prom_stale": PromStaleEvent, + "prom_window_cross_event": PromWindowCrossEvent, + "budget_change": BudgetChangeEvent, + "frontend_post_fault": FrontendPostFaultEvent, + "mdc_unavailable": MdcUnavailableEvent, + "aic_failure": AicFailureEvent, + } + cls = type_map.get(etype) + if cls is None: + raise ValueError(f"Unknown event type: {etype!r}") + result.append(cls(**e)) + return result + + def parsed_assertions(self) -> list[Assertion]: + """Parse raw assertion dicts into typed Assertion objects.""" + result = [] + for a in self.assertions: + if "expr" in a: + result.append(ExprAssertion(**a)) + else: + result.append(StructuredAssertion(**a)) + return result + + +# --------------------------------------------------------------------------- +# YAML loader with ``extends:`` support +# --------------------------------------------------------------------------- + +_SCENARIOS_DIR = Path(__file__).parent / "scenarios" + + +def _deep_merge(base: dict, override: dict) -> dict: + """Recursive dict merge: override keys win; missing keys inherit from base. + + Lists are REPLACED (not concatenated) — makes scenarios easy to read. + """ + result = dict(base) + for k, v in override.items(): + if k in result and isinstance(result[k], dict) and isinstance(v, dict): + result[k] = _deep_merge(result[k], v) + else: + result[k] = v + return result + + +def load_scenario(path: Union[str, Path]) -> ScenarioSpec: + """Load a scenario YAML file, resolving ``extends:`` inheritance. + + The ``extends:`` key must point to a path relative to the ``scenarios/`` + directory root. Only a single level of inheritance is supported (the + base template cannot itself ``extends:`` another file). + """ + path = Path(path) + with path.open() as f: + raw: dict[str, Any] = yaml.safe_load(f) or {} + + extends = raw.pop("extends", None) + if extends: + base_path = _SCENARIOS_DIR / extends + with base_path.open() as f: + base_raw: dict[str, Any] = yaml.safe_load(f) or {} + base_raw.pop("extends", None) # base cannot chain + raw = _deep_merge(base_raw, raw) + + # Rename Python-reserved alias: "class" → "class_" via Pydantic alias + return ScenarioSpec.model_validate(raw) + + +def load_all_scenarios() -> list[tuple[str, ScenarioSpec]]: + """Load all scenario YAML files from the scenarios/ directory. + + Returns list of (scenario_name, ScenarioSpec). Skips _base/ templates. + """ + results = [] + for yaml_path in sorted(_SCENARIOS_DIR.glob("*.yaml")): + spec = load_scenario(yaml_path) + results.append((spec.name, spec)) + return results + + +def validate_assertion_fields(assertions: list[dict[str, Any]]) -> list[str]: + """Validate field / ref / expr references in assertions. + + Returns a list of error messages (empty list means all valid). + """ + errors: list[str] = [] + known_fields = _tick_snapshot_fields() + for i, a in enumerate(assertions): + field_name = a.get("field") + if field_name and field_name not in known_fields: + errors.append( + f"assertion[{i}]: field {field_name!r} not in TickSnapshot. " + f"Known fields: {sorted(known_fields)}" + ) + ref = a.get("ref") + if ref: + parts = ref.split(".") + if parts[0] not in ("planner", "counters", "fleet", "overlay"): + errors.append( + f"assertion[{i}]: ref {ref!r} must start with " + f"'planner.', 'counters.', 'fleet.', or 'overlay.'" + ) + expr = a.get("expr") + if expr: + try: + tree = ast.parse(expr, mode="eval") + _validate_expr_node(tree, i, errors) + except SyntaxError as e: + errors.append(f"assertion[{i}]: expr syntax error: {e}") + return errors + + +def _validate_expr_node(tree: ast.AST, idx: int, errors: list[str]) -> None: + """Walk expression AST and reject unknown history/planner/counters references.""" + _ALLOWED_ROOTS = {"history", "planner", "counters", "abs", "min", "max"} + for node in ast.walk(tree): + if isinstance(node, ast.Name) and node.id not in _ALLOWED_ROOTS: + errors.append( + f"assertion[{idx}]: expr references unknown name {node.id!r}; " + f"allowed roots: {_ALLOWED_ROOTS}" + ) + + +class ScenarioLoadError(Exception): + pass diff --git a/components/src/dynamo/planner/tests/testbed/scenarios/A1_power_under_estimate_decode.yaml b/components/src/dynamo/planner/tests/testbed/scenarios/A1_power_under_estimate_decode.yaml new file mode 100644 index 000000000000..a02a1f255978 --- /dev/null +++ b/components/src/dynamo/planner/tests/testbed/scenarios/A1_power_under_estimate_decode.yaml @@ -0,0 +1,25 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +extends: _base/h200_disagg.yaml +name: A1_power_under_estimate_decode +description: > + Truth-side decode power is 1.35× AIC's prediction from t=0. Validates that + c_power_d EMA converges to ~1.35 and cap_d is re-inflated on next sweep. + +fleet: + bias: + power_bias_decode: 1.35 + +assertions: + - at_tick: 50 + field: c_power_d + op: "within" + value: 1.35 + tolerance: 0.15 + description: "c_power_d converges toward 1.35 by tick 50" + - always: true + field: projected_w + op: "<=" + ref: "planner.total_gpu_power_limit" + description: "Budget never exceeded" diff --git a/components/src/dynamo/planner/tests/testbed/scenarios/A2_power_over_estimate_prefill.yaml b/components/src/dynamo/planner/tests/testbed/scenarios/A2_power_over_estimate_prefill.yaml new file mode 100644 index 000000000000..f89a141a86be --- /dev/null +++ b/components/src/dynamo/planner/tests/testbed/scenarios/A2_power_over_estimate_prefill.yaml @@ -0,0 +1,28 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +extends: _base/h200_disagg.yaml +name: A2_power_over_estimate_prefill +description: > + Truth-side prefill power is 0.7× AIC's prediction. Validates that c_power_p + EMA shrinks but max(1.0, c) asymmetric clamp holds — cap_p unchanged. + +fleet: + bias: + power_bias_prefill: 0.70 + +assertions: + - at_tick: 100 + field: c_power_p + op: "<" + value: 1.0 + description: "c_power_p shrinks below 1.0 (raw EMA tracks actual draw)" + - always: true + field: cap_p + op: ">=" + value: 450 + description: "cap_p not reduced below AIC prediction (asymmetric clamp)" + - always: true + field: projected_w + op: "<=" + ref: "planner.total_gpu_power_limit" diff --git a/components/src/dynamo/planner/tests/testbed/scenarios/A3_ttft_under_estimate.yaml b/components/src/dynamo/planner/tests/testbed/scenarios/A3_ttft_under_estimate.yaml new file mode 100644 index 000000000000..9e73b73669e4 --- /dev/null +++ b/components/src/dynamo/planner/tests/testbed/scenarios/A3_ttft_under_estimate.yaml @@ -0,0 +1,29 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +extends: _base/h200_disagg.yaml +name: A3_ttft_under_estimate +description: > + TTFT is 1.4× AIC prediction and ITL is 1.3×. SLA-miss trigger fires; + after aic_drift_consecutive_ticks a re-sweep occurs. + +fleet: + bias: + # 1.4 was the original design value but produces observed TTFT ≈ 261ms + # after the cap-factor inflation (cap_p=450 against tdp=700) which is + # well under the 500ms SLA — no drift trigger. 3.0 puts observed TTFT + # at ≈ 560ms, comfortably above SLA so the drift-consecutive counter + # actually advances and a re-sweep fires within ``eventually_by_tick``. + ttft_bias: 3.0 + itl_bias: 1.3 + +assertions: + - eventually_by_tick: 50 + field: sweep_fired + op: "==" + value: 1.0 + description: "Optimizer re-sweeps after sustained SLA violation" + - always: true + field: projected_w + op: "<=" + ref: "planner.total_gpu_power_limit" diff --git a/components/src/dynamo/planner/tests/testbed/scenarios/A4_step_drift_midstream.yaml b/components/src/dynamo/planner/tests/testbed/scenarios/A4_step_drift_midstream.yaml new file mode 100644 index 000000000000..ea5b2cd34548 --- /dev/null +++ b/components/src/dynamo/planner/tests/testbed/scenarios/A4_step_drift_midstream.yaml @@ -0,0 +1,31 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +extends: _base/h200_disagg.yaml +name: A4_step_drift_midstream +description: > + Decode power bias steps from 1.0 to 1.35 at tick 100. EMA half-life ≈ 5 + ticks (α=0.3). Verifies hysteresis prevents immediate re-sweep flap. + +events: + - type: bias_step + at_tick: 100 + signal: power_bias_decode + value: 1.35 + +assertions: + - at_tick: 80 + field: c_power_d + op: "within" + value: 1.0 + tolerance: 0.12 + description: "Pre-step c_power_d near 1.0" + - at_tick: 150 + field: c_power_d + op: ">" + value: 1.1 + description: "Post-step c_power_d has risen tracking the bias" + - always: true + field: projected_w + op: "<=" + ref: "planner.total_gpu_power_limit" diff --git a/components/src/dynamo/planner/tests/testbed/scenarios/A5_oscillating_drift_sub_interval.yaml b/components/src/dynamo/planner/tests/testbed/scenarios/A5_oscillating_drift_sub_interval.yaml new file mode 100644 index 000000000000..9e72e5589ede --- /dev/null +++ b/components/src/dynamo/planner/tests/testbed/scenarios/A5_oscillating_drift_sub_interval.yaml @@ -0,0 +1,27 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +extends: _base/h200_disagg.yaml +name: A5_oscillating_drift_sub_interval +description: > + Decode power bias oscillates with period 4 ticks (< aic_reoptimize_interval + of 5 ticks). No sweep storm; mean coefficient stays near 1.0 ± noise band. + +events: + - type: bias_sine + signal: power_bias_decode + amplitude: 0.20 + period_ticks: 4 + offset: 0.0 + +assertions: + - at_tick: 100 + field: c_power_d + op: "within" + value: 1.0 + tolerance: 0.35 + description: "Mean c_power_d stays near 1.0 despite oscillation" + - always: true + field: projected_w + op: "<=" + ref: "planner.total_gpu_power_limit" diff --git a/components/src/dynamo/planner/tests/testbed/scenarios/A6_coefficient_pegged_at_clamp.yaml b/components/src/dynamo/planner/tests/testbed/scenarios/A6_coefficient_pegged_at_clamp.yaml new file mode 100644 index 000000000000..f39fe121bf7c --- /dev/null +++ b/components/src/dynamo/planner/tests/testbed/scenarios/A6_coefficient_pegged_at_clamp.yaml @@ -0,0 +1,24 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +extends: _base/h200_disagg.yaml +name: A6_coefficient_pegged_at_clamp +description: > + Decode power bias = 3.0 (far above AIC prediction). Coefficient saturates + at clamp of 2.0. Planner does NOT auto-disable; pegged counter increments. + +fleet: + bias: + power_bias_decode: 3.0 + +assertions: + - at_tick: 50 + field: c_power_d + op: "within" + value: 2.0 + tolerance: 0.05 + description: "c_power_d pegged at clamp=2.0" + - always: true + field: projected_w + op: "<=" + ref: "planner.total_gpu_power_limit" diff --git a/components/src/dynamo/planner/tests/testbed/scenarios/B10_daemonset_absent_one_node.yaml b/components/src/dynamo/planner/tests/testbed/scenarios/B10_daemonset_absent_one_node.yaml new file mode 100644 index 000000000000..e96368c31c8d --- /dev/null +++ b/components/src/dynamo/planner/tests/testbed/scenarios/B10_daemonset_absent_one_node.yaml @@ -0,0 +1,29 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +extends: _base/h200_disagg.yaml +name: B10_daemonset_absent_one_node +description: > + DaemonSet absent fault: annotation is recorded but not reflected to truth + model. Truth-side draw = TDP × n; c_power_d climbs; re-sweep fires or pegs. + +events: + - type: actuation_fault + at_tick: 0 + duration_ticks: 200 + mode: daemonset_absent + +fleet: + bias: + power_bias_decode: 1.5 + +assertions: + - eventually_by_tick: 100 + field: c_power_d + op: ">" + value: 1.2 + description: "c_power_d drifts up when cap not applied to truth model" + - always: true + field: projected_w + op: "<=" + ref: "planner.total_gpu_power_limit" diff --git a/components/src/dynamo/planner/tests/testbed/scenarios/B11_frontend_post_partial_failure.yaml b/components/src/dynamo/planner/tests/testbed/scenarios/B11_frontend_post_partial_failure.yaml new file mode 100644 index 000000000000..206ed4f0f8ed --- /dev/null +++ b/components/src/dynamo/planner/tests/testbed/scenarios/B11_frontend_post_partial_failure.yaml @@ -0,0 +1,32 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +extends: _base/h200_disagg.yaml +name: B11_frontend_post_partial_failure +description: > + 2/3 of frontend POSTs return 503 for 10 ticks. admission_partial_success_total + increments. _apply_aic_config does not roll back; next tick re-fans-out. + +planner: + admission_mode: autoset + +events: + # Fault active from startup so the cold-start /busy_threshold fan-out + # (which is the only POST fan-out in a no-sweep scenario) hits it. With + # failing_fraction=0.67 and 3 synthetic frontend pods the Bernoulli + # probability of zero failures is (1-0.67)^3 ≈ 0.036 — vanishingly small. + - type: frontend_post_fault + at_tick: 0 + duration_ticks: 30 + failing_fraction: 0.67 + +assertions: + - eventually_by_tick: 35 + field: admission_partial_failures + op: ">" + value: 0 + description: "Partial POST failure counter increments during fault window" + - always: true + field: projected_w + op: "<=" + ref: "planner.total_gpu_power_limit" diff --git a/components/src/dynamo/planner/tests/testbed/scenarios/B7_nvml_clamp_low.yaml b/components/src/dynamo/planner/tests/testbed/scenarios/B7_nvml_clamp_low.yaml new file mode 100644 index 000000000000..89d79326a934 --- /dev/null +++ b/components/src/dynamo/planner/tests/testbed/scenarios/B7_nvml_clamp_low.yaml @@ -0,0 +1,29 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +extends: _base/h200_disagg.yaml +name: B7_nvml_clamp_low +description: > + Actuation fault nvml_low: optimizer requests 180W decode cap (below H200 + minimum of 200W). FakeActuator clamps to 200W. Power agent cap_clamped_min + counter increments; truth-side draw = 200W, c_power_d drifts up. + +planner: + decode_engine_gpu_power_limit: 180 + +events: + - type: actuation_fault + at_tick: 0 + duration_ticks: 200 + mode: nvml_low + +assertions: + - at_tick: 10 + field: cap_clamped_min + op: ">" + value: 0 + description: "SKU min clamp counter increments" + - always: true + field: projected_w + op: "<=" + ref: "planner.total_gpu_power_limit" diff --git a/components/src/dynamo/planner/tests/testbed/scenarios/B8_nvml_clamp_high.yaml b/components/src/dynamo/planner/tests/testbed/scenarios/B8_nvml_clamp_high.yaml new file mode 100644 index 000000000000..b596ce0e46e6 --- /dev/null +++ b/components/src/dynamo/planner/tests/testbed/scenarios/B8_nvml_clamp_high.yaml @@ -0,0 +1,29 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +extends: _base/h100_pcie_disagg.yaml +name: B8_nvml_clamp_high +description: > + H100 PCIe with TDP=350W. Optimizer picks 750W cap (above SKU max). + FakeActuator clamps to 350W; cap_clamped_max counter increments. + +planner: + prefill_engine_gpu_power_limit: 750 + decode_engine_gpu_power_limit: 750 + +events: + - type: actuation_fault + at_tick: 0 + duration_ticks: 200 + mode: nvml_high + +assertions: + - at_tick: 5 + field: cap_clamped_max + op: ">" + value: 0 + description: "SKU max clamp counter increments" + - always: true + field: projected_w + op: "<=" + ref: "planner.total_gpu_power_limit" diff --git a/components/src/dynamo/planner/tests/testbed/scenarios/B9_k8s_rbac_denied.yaml b/components/src/dynamo/planner/tests/testbed/scenarios/B9_k8s_rbac_denied.yaml new file mode 100644 index 000000000000..54edc6afe052 --- /dev/null +++ b/components/src/dynamo/planner/tests/testbed/scenarios/B9_k8s_rbac_denied.yaml @@ -0,0 +1,20 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +extends: _base/h200_disagg.yaml +name: B9_k8s_rbac_denied +description: > + RBAC denied fault for 5 ticks. WARNING should be logged; scaling decisions + not blocked. Power annotations still in config even though apply fails. + +events: + - type: actuation_fault + at_tick: 10 + duration_ticks: 5 + mode: rbac_denied + +assertions: + - always: true + field: projected_w + op: "<=" + ref: "planner.total_gpu_power_limit" diff --git a/components/src/dynamo/planner/tests/testbed/scenarios/C12_one_node_down_low_load.yaml b/components/src/dynamo/planner/tests/testbed/scenarios/C12_one_node_down_low_load.yaml new file mode 100644 index 000000000000..136f4cb882e3 --- /dev/null +++ b/components/src/dynamo/planner/tests/testbed/scenarios/C12_one_node_down_low_load.yaml @@ -0,0 +1,29 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +extends: _base/h200_disagg.yaml +name: C12_one_node_down_low_load +description: > + One decode replica lost; offered load < remaining capacity. + No sweep fires (capacity_exceeded=false, no SLA miss) — validates + "Direction matters" from §5.6. + +load: + profile: constant + tokens_per_sec: 500.0 + +events: + - type: node_down + at_tick: 50 + n_decode_lost: 1 + +assertions: + - at_tick: 80 + field: sweep_fired + op: "==" + value: 0.0 + description: "No re-sweep when under-loaded after node loss" + - always: true + field: projected_w + op: "<=" + ref: "planner.total_gpu_power_limit" diff --git a/components/src/dynamo/planner/tests/testbed/scenarios/C13_one_node_down_high_load.yaml b/components/src/dynamo/planner/tests/testbed/scenarios/C13_one_node_down_high_load.yaml new file mode 100644 index 000000000000..0bb7296ad92e --- /dev/null +++ b/components/src/dynamo/planner/tests/testbed/scenarios/C13_one_node_down_high_load.yaml @@ -0,0 +1,33 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +extends: _base/h200_disagg.yaml +name: C13_one_node_down_high_load +description: > + One decode replica lost at high load. SLA-miss path fires after + aic_drift_consecutive_ticks; re-sweep to higher-replica config. + +load: + profile: constant + # Synthetic fleet capacity at post-clamp (1P, 4D) is ≈ 660k tok/s. 3500 was + # the original design value but produced ~0.5% utilization — no queue + # saturation, no TTFT inflation, no drift trigger after the node_down. + # 700k saturates the (1P, 3D) post-node-down configuration enough for + # queue_factor to push observed TTFT past the 500ms SLA. + tokens_per_sec: 700000.0 + +events: + - type: node_down + at_tick: 30 + n_decode_lost: 1 + +assertions: + - eventually_by_tick: 80 + field: sweep_fired + op: "==" + value: 1.0 + description: "Re-sweep fires after SLA miss on high load" + - always: true + field: projected_w + op: "<=" + ref: "planner.total_gpu_power_limit" diff --git a/components/src/dynamo/planner/tests/testbed/scenarios/C14_all_decode_workers_fail.yaml b/components/src/dynamo/planner/tests/testbed/scenarios/C14_all_decode_workers_fail.yaml new file mode 100644 index 000000000000..28b56f4325dd --- /dev/null +++ b/components/src/dynamo/planner/tests/testbed/scenarios/C14_all_decode_workers_fail.yaml @@ -0,0 +1,35 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +extends: _base/h200_disagg.yaml +name: C14_all_decode_workers_fail +description: > + All decode replicas fail. total_tokens_per_sec ≈ 0 does NOT trigger drift + (correct, by design); SLA-miss (TTFT timeout) triggers re-sweep attempt. + +events: + - type: node_down + at_tick: 20 + n_decode_lost: 4 + +assertions: + # After ``node_down`` at tick 20, ``n_d_truth`` immediately drops to 0. + # The drift detector then sees zero capacity → SLA miss → re-sweep at + # roughly tick 25, which recreates decode replicas. So "all decode gone" + # is observable in the one-tick window between the failure and the next + # re-sweep — at_tick: 21 captures that window exactly. + - at_tick: 21 + field: n_d + op: "==" + value: 0.0 + description: "All decode replicas gone immediately after node failure" + # The planner SHOULD react with a re-sweep — fire by tick 35. + - eventually_by_tick: 35 + field: sweep_fired + op: "==" + value: 1.0 + description: "Re-sweep fires after capacity drops to zero" + - always: true + field: projected_w + op: "<=" + ref: "planner.total_gpu_power_limit" diff --git a/components/src/dynamo/planner/tests/testbed/scenarios/C15_node_recovery.yaml b/components/src/dynamo/planner/tests/testbed/scenarios/C15_node_recovery.yaml new file mode 100644 index 000000000000..fb803d3fd5b8 --- /dev/null +++ b/components/src/dynamo/planner/tests/testbed/scenarios/C15_node_recovery.yaml @@ -0,0 +1,34 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +extends: _base/h200_disagg.yaml +name: C15_node_recovery +description: > + Node down at tick 50, node up at tick 100. No permanent state corruption; + coefficients re-converge; final config matches pre-fault steady state. + +events: + - type: node_down + at_tick: 50 + n_decode_lost: 2 + - type: node_up + at_tick: 100 + n_decode_restored: 2 + +assertions: + - at_tick: 45 + field: c_power_d + op: "within" + value: 1.0 + tolerance: 0.20 + description: "Pre-fault c_power_d near 1.0" + - at_tick: 180 + field: c_power_d + op: "within" + value: 1.0 + tolerance: 0.30 + description: "Post-recovery c_power_d re-converges" + - always: true + field: projected_w + op: "<=" + ref: "planner.total_gpu_power_limit" diff --git a/components/src/dynamo/planner/tests/testbed/scenarios/C16_warmup_power_spike.yaml b/components/src/dynamo/planner/tests/testbed/scenarios/C16_warmup_power_spike.yaml new file mode 100644 index 000000000000..1a6b70dc3b7d --- /dev/null +++ b/components/src/dynamo/planner/tests/testbed/scenarios/C16_warmup_power_spike.yaml @@ -0,0 +1,40 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +extends: _base/h200_disagg.yaml +name: C16_warmup_power_spike +description: > + New replica draws 0.7×TDP for first 15 simulated seconds (warmup period), + then settles at AIC-predicted. Models §6.5 warmup power concern. Validates + simultaneous-restart budget headroom (headroom_factor=0.85). + +ticks: 60 + +fleet: + bias: + power_bias_prefill: 1.0 + power_bias_decode: 1.0 + +events: + - type: node_up + at_tick: 5 + n_decode_restored: 2 + - type: bias_step + at_tick: 5 + signal: power_bias_decode + value: 0.7 + - type: bias_step + at_tick: 8 + signal: power_bias_decode + value: 1.0 + +assertions: + - at_tick: 6 + field: observed_power_w_d + op: "<" + value: 400.0 + description: "Warmup: decode power below steady-state during warmup" + - always: true + field: projected_w + op: "<=" + ref: "planner.total_gpu_power_limit" diff --git a/components/src/dynamo/planner/tests/testbed/scenarios/D17_prometheus_outage.yaml b/components/src/dynamo/planner/tests/testbed/scenarios/D17_prometheus_outage.yaml new file mode 100644 index 000000000000..099e6a828798 --- /dev/null +++ b/components/src/dynamo/planner/tests/testbed/scenarios/D17_prometheus_outage.yaml @@ -0,0 +1,26 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +extends: _base/h200_disagg.yaml +name: D17_prometheus_outage +description: > + Prometheus outage for 10 ticks (all signals). EMA gates skip (None returns); + coefficients held at prior values. Budget enforcement unaffected. + +events: + - type: prom_outage + at_tick: 30 + duration_ticks: 10 + signals: ["ttft", "itl", "power_p", "power_d", "capacity"] + +assertions: + - at_tick: 35 + field: c_power_d + op: "within" + value: 1.0 + tolerance: 0.20 + description: "EMA held at prior value during outage (no update)" + - always: true + field: projected_w + op: "<=" + ref: "planner.total_gpu_power_limit" diff --git a/components/src/dynamo/planner/tests/testbed/scenarios/D18_prometheus_stale.yaml b/components/src/dynamo/planner/tests/testbed/scenarios/D18_prometheus_stale.yaml new file mode 100644 index 000000000000..ab104c3a2220 --- /dev/null +++ b/components/src/dynamo/planner/tests/testbed/scenarios/D18_prometheus_stale.yaml @@ -0,0 +1,30 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +extends: _base/h200_disagg.yaml +name: D18_prometheus_stale +description: > + Prometheus stale with lag_ticks=5 for 20 ticks. EMA tracks lagged signal; + no overshoot/oscillation. Verifies EMA + hysteresis composition. + +events: + - type: bias_step + at_tick: 20 + signal: power_bias_decode + value: 1.3 + - type: prom_stale + at_tick: 20 + duration_ticks: 20 + lag_ticks: 5 + +assertions: + - at_tick: 45 + field: c_power_d + op: "within" + value: 1.15 + tolerance: 0.20 + description: "Lagged EMA converges to midpoint (5-tick lag on step)" + - always: true + field: projected_w + op: "<=" + ref: "planner.total_gpu_power_limit" diff --git a/components/src/dynamo/planner/tests/testbed/scenarios/D19_dcgm_attribution_loss.yaml b/components/src/dynamo/planner/tests/testbed/scenarios/D19_dcgm_attribution_loss.yaml new file mode 100644 index 000000000000..fab5825be27e --- /dev/null +++ b/components/src/dynamo/planner/tests/testbed/scenarios/D19_dcgm_attribution_loss.yaml @@ -0,0 +1,26 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +extends: _base/h200_disagg.yaml +name: D19_dcgm_attribution_loss +description: > + Prometheus outage on power_p and power_d only (DCGM attribution loss). + Power EMA gates fail; latency coefficients still update. + +events: + - type: prom_outage + at_tick: 20 + duration_ticks: 30 + signals: ["power_p", "power_d"] + +assertions: + - at_tick: 35 + field: c_power_d + op: "within" + value: 1.0 + tolerance: 0.10 + description: "Power EMA held during outage" + - always: true + field: projected_w + op: "<=" + ref: "planner.total_gpu_power_limit" diff --git a/components/src/dynamo/planner/tests/testbed/scenarios/D20_mdc_missing_max_batched_tokens.yaml b/components/src/dynamo/planner/tests/testbed/scenarios/D20_mdc_missing_max_batched_tokens.yaml new file mode 100644 index 000000000000..56d8b8719490 --- /dev/null +++ b/components/src/dynamo/planner/tests/testbed/scenarios/D20_mdc_missing_max_batched_tokens.yaml @@ -0,0 +1,22 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +extends: _base/h200_disagg.yaml +name: D20_mdc_missing_max_batched_tokens +description: > + MDC unavailable for 20 ticks. admission_max_batched_tokens_unavailable_total + increments; fractional threshold still applied; absolute threshold skipped. + +planner: + admission_mode: autoset + +events: + - type: mdc_unavailable + at_tick: 10 + duration_ticks: 20 + +assertions: + - always: true + field: projected_w + op: "<=" + ref: "planner.total_gpu_power_limit" diff --git a/components/src/dynamo/planner/tests/testbed/scenarios/D21_prom_window_cross_after_cap_change.yaml b/components/src/dynamo/planner/tests/testbed/scenarios/D21_prom_window_cross_after_cap_change.yaml new file mode 100644 index 000000000000..b1c7f5ab7a8f --- /dev/null +++ b/components/src/dynamo/planner/tests/testbed/scenarios/D21_prom_window_cross_after_cap_change.yaml @@ -0,0 +1,42 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +extends: _base/h200_disagg.yaml +name: D21_prom_window_cross_after_cap_change +description: > + Optimizer sets cap_d 425→200W at tick 50; a prom_window_cross_event fires + at tick 51 with weight_old=0.66 (mixing pre-cap and post-cap observations). + Verifies that the controller's EMA + hysteresis do NOT permanently spike + c_power_d above 1.5; should re-center on 1.0 within ~5 ticks. + +planner: + decode_engine_gpu_power_limit: 200 + +events: + - type: bias_step + at_tick: 50 + signal: power_bias_decode + value: 0.55 + auto_inject_window_cross: true + +assertions: + - at_tick: 53 + field: c_power_d + op: "<=" + value: 2.0 + description: "No permanent overshoot past clamp after window-cross tick" + # The bias-step persists for the rest of the run, so the EMA correctly + # converges to the new bias value (0.55), not 1.0. The original assertion + # incorrectly expected re-centering on 1.0 — that was based on a mental + # model where the bias was transient. We assert the EMA settles near + # 0.55 (within 30%) instead. + - at_tick: 80 + field: c_power_d + op: "within" + value: 0.55 + tolerance: 0.30 + description: "c_power_d converges to the new bias (no spike, no permanent overshoot)" + - always: true + field: projected_w + op: "<=" + ref: "planner.total_gpu_power_limit" diff --git a/components/src/dynamo/planner/tests/testbed/scenarios/E21_budget_shrunk_live.yaml b/components/src/dynamo/planner/tests/testbed/scenarios/E21_budget_shrunk_live.yaml new file mode 100644 index 000000000000..62f63ddd6689 --- /dev/null +++ b/components/src/dynamo/planner/tests/testbed/scenarios/E21_budget_shrunk_live.yaml @@ -0,0 +1,24 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +extends: _base/h200_disagg.yaml +name: E21_budget_shrunk_live +description: > + Budget shrinks 4000→2000W at tick 50. _apply_power_budget clamps replicas + immediately. AIC re-sweeps. No oscillation after initial clamp. + +events: + - type: budget_change + at_tick: 50 + new_total_w: 2000 + +assertions: + - at_tick: 55 + field: projected_w + op: "<=" + value: 2000 + description: "Projected power clamps to new budget within a few ticks" + - always: true + field: projected_w + op: "<=" + ref: "planner.total_gpu_power_limit" diff --git a/components/src/dynamo/planner/tests/testbed/scenarios/E22_budget_below_min_endpoint.yaml b/components/src/dynamo/planner/tests/testbed/scenarios/E22_budget_below_min_endpoint.yaml new file mode 100644 index 000000000000..57f73ed89b65 --- /dev/null +++ b/components/src/dynamo/planner/tests/testbed/scenarios/E22_budget_below_min_endpoint.yaml @@ -0,0 +1,19 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +extends: _base/h200_disagg.yaml +name: E22_budget_below_min_endpoint +description: > + Budget is too small to cover even min_endpoint replicas from tick 0. + _apply_power_budget returns (0, 0) with WARNING; static static enforcement + still functions. + +planner: + total_gpu_power_limit: 100 + +assertions: + - at_tick: 5 + field: n_d + op: "==" + value: 0.0 + description: "Budget too small → zero decode replicas" diff --git a/components/src/dynamo/planner/tests/testbed/scenarios/E23_aic_infeasible_at_startup.yaml b/components/src/dynamo/planner/tests/testbed/scenarios/E23_aic_infeasible_at_startup.yaml new file mode 100644 index 000000000000..78b9d2d45a07 --- /dev/null +++ b/components/src/dynamo/planner/tests/testbed/scenarios/E23_aic_infeasible_at_startup.yaml @@ -0,0 +1,21 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +extends: _base/h200_disagg.yaml +name: E23_aic_infeasible_at_startup +description: > + AIC returns empty/infeasible Pareto at startup (tick 0). Optimizer + auto-disables with reason="infeasible_at_startup". Static + _apply_power_budget still functions. + +events: + - type: aic_failure + at_tick: 0 + mode: empty_pareto + n_consecutive: 200 + +assertions: + - always: true + field: projected_w + op: "<=" + ref: "planner.total_gpu_power_limit" diff --git a/components/src/dynamo/planner/tests/testbed/scenarios/E24_aic_exception_at_runtime.yaml b/components/src/dynamo/planner/tests/testbed/scenarios/E24_aic_exception_at_runtime.yaml new file mode 100644 index 000000000000..b1727f5ff597 --- /dev/null +++ b/components/src/dynamo/planner/tests/testbed/scenarios/E24_aic_exception_at_runtime.yaml @@ -0,0 +1,25 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +extends: _base/h200_disagg.yaml +name: E24_aic_exception_at_runtime +description: > + AIC raises at tick 50 (after a prior successful sweep). _last_optimal_config + is retained; aic_optimizer_exceptions_total increments; planner stays alive. + +events: + - type: aic_failure + at_tick: 50 + mode: raises + n_consecutive: 1 + +assertions: + - at_tick: 52 + field: optimizer_exceptions + op: ">=" + value: 0.0 + description: "Planner remains alive after exception" + - always: true + field: projected_w + op: "<=" + ref: "planner.total_gpu_power_limit" diff --git a/components/src/dynamo/planner/tests/testbed/scenarios/E25_aic_5_consecutive_failures.yaml b/components/src/dynamo/planner/tests/testbed/scenarios/E25_aic_5_consecutive_failures.yaml new file mode 100644 index 000000000000..8b88f0a23145 --- /dev/null +++ b/components/src/dynamo/planner/tests/testbed/scenarios/E25_aic_5_consecutive_failures.yaml @@ -0,0 +1,25 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +extends: _base/h200_disagg.yaml +name: E25_aic_5_consecutive_failures +description: > + AIC raises for 5 consecutive sweeps (n_consecutive=5). After the 5th, + optimizer is auto-disabled; aic_consecutive_failures gauge reaches 5 and + then transitions to disabled state. + +planner: + aic_max_consecutive_failures: 5 + aic_reoptimize_interval: 60 + +events: + - type: aic_failure + at_tick: 10 + mode: raises + n_consecutive: 5 + +assertions: + - always: true + field: projected_w + op: "<=" + ref: "planner.total_gpu_power_limit" diff --git a/components/src/dynamo/planner/tests/testbed/scenarios/F26_drift_threshold_boundary.yaml b/components/src/dynamo/planner/tests/testbed/scenarios/F26_drift_threshold_boundary.yaml new file mode 100644 index 000000000000..c8f5719319b8 --- /dev/null +++ b/components/src/dynamo/planner/tests/testbed/scenarios/F26_drift_threshold_boundary.yaml @@ -0,0 +1,55 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +extends: _base/h200_disagg.yaml +name: F26_drift_threshold_boundary +description: > + Drift-detector boundary test re-targeted at the SLA-miss path + (the capacity_exceeded path is unreachable with the current truth + fleet, since synthetic ``throughput = min(offered_load, capacity)`` + and ``estimated_throughput`` is anchored to the post-clamp replica + count — so processed traffic can never exceed the optimizer's + prediction by 15%, see runner.py setup notes). + + Truth-side TTFT is ttft_bias × cap_factor × aic_ttft_ms. + With cap_p=450 and tdp=700, cap_factor ≈ 1.556 and aic_ttft_ms=120. + So observed TTFT ≈ ttft_bias × 187 ms. SLA is 500 ms. + +ticks: 60 +planner: + aic_drift_relative_threshold: 0.15 + aic_drift_consecutive_ticks: 3 + aic_reoptimize_interval: 60 + +fleet: + bias: + # 2.5 × 187 ≈ 467 ms — just below 500 ms SLA, no trigger. + ttft_bias: 2.5 + +events: + # Step bias up at tick 20 so observed TTFT crosses SLA. With + # aic_drift_consecutive_ticks=3 the sweep should fire at tick 23. + - type: bias_step + at_tick: 20 + signal: ttft_bias + value: 3.5 # 3.5 × 187 ≈ 654 ms — comfortably over 500 ms SLA. + +load: + profile: constant + tokens_per_sec: 2000.0 + +assertions: + - at_tick: 18 + field: sweep_fired + op: "==" + value: 0.0 + description: "No sweep while observed TTFT below SLA (bias × cap_factor < 500 ms)" + - eventually_by_tick: 55 + field: sweep_fired + op: "==" + value: 1.0 + description: "Sweep fires after bias step pushes TTFT above SLA for ≥3 consecutive ticks" + - always: true + field: projected_w + op: "<=" + ref: "planner.total_gpu_power_limit" diff --git a/components/src/dynamo/planner/tests/testbed/scenarios/G1_realistic_decode_drift.yaml b/components/src/dynamo/planner/tests/testbed/scenarios/G1_realistic_decode_drift.yaml new file mode 100644 index 000000000000..6a68ea94e634 --- /dev/null +++ b/components/src/dynamo/planner/tests/testbed/scenarios/G1_realistic_decode_drift.yaml @@ -0,0 +1,32 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +extends: _base/mocker_h200_disagg.yaml +name: G1_realistic_decode_drift +description: > + Mocker replay + decode power bias 1.30 + one nvml_low fault at tick 30. + Validates controller closes the loop when load arrives from a real scheduler. + Latency and capacity from real mocker scheduling; power from overlay. + Pairs with A1 (same bias, real scheduler instead of synthetic fleet). + +overlay: + bias: + power_bias_decode: 1.30 + +events: + - type: actuation_fault + at_tick: 30 + duration_ticks: 5 + mode: nvml_low + +assertions: + - at_tick: 25 + field: c_power_d + op: "within" + value: 1.30 + tolerance: 0.25 + description: "c_power_d converges toward 1.30 with real mocker load (looser than α)" + - always: true + field: projected_w + op: "<=" + ref: "planner.total_gpu_power_limit" diff --git a/components/src/dynamo/planner/tests/testbed/scenarios/G2_scheduler_driven_scale_out.yaml b/components/src/dynamo/planner/tests/testbed/scenarios/G2_scheduler_driven_scale_out.yaml new file mode 100644 index 000000000000..aa9e55438a92 --- /dev/null +++ b/components/src/dynamo/planner/tests/testbed/scenarios/G2_scheduler_driven_scale_out.yaml @@ -0,0 +1,30 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +extends: _base/mocker_h200_disagg.yaml +name: G2_scheduler_driven_scale_out +description: > + Mocker replay with synthetic load ramp (tokens_per_sec: 100→800 over 60 s). + Validates that AIC optimizer and scheduler jointly expand decode pool within + budget and without oscillation. Tests the interface between scheduler + dynamics and the power budget: n_d should increase, projected_w should stay + under budget. + +overlay: + bias: {} + +assertions: + - at_tick: 30 + field: n_d + op: ">=" + value: 1.0 + description: "Scheduler has added at least one decode replica by tick 30" + - always: true + field: projected_w + op: "<=" + ref: "planner.total_gpu_power_limit" + - always: true + field: n_oscillations + op: "<=" + value: 3.0 + description: "At most 3 replica flip-flops over the entire run" diff --git a/components/src/dynamo/planner/tests/testbed/scenarios/G3_scheduler_power_cap_interaction.yaml b/components/src/dynamo/planner/tests/testbed/scenarios/G3_scheduler_power_cap_interaction.yaml new file mode 100644 index 000000000000..3ea61614a998 --- /dev/null +++ b/components/src/dynamo/planner/tests/testbed/scenarios/G3_scheduler_power_cap_interaction.yaml @@ -0,0 +1,30 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +extends: _base/mocker_h200_synthetic_workload.yaml +name: G3_scheduler_power_cap_interaction +description: > + Mocker replay with sustained high load (tokens_per_sec: 1000, full fleet). + Budget is deliberately tight (3600W for 4-node H200). AIC must reduce + per-GPU power cap; overlay ensures power observability. Validates that + power caps are applied, observed, and that caps do not contradict scheduler + scaling decisions. + +planner: + total_gpu_power_limit: 3600 + +overlay: + bias: + power_bias_prefill: 1.05 + power_bias_decode: 1.05 + +assertions: + - at_tick: 20 + field: cap_d + op: "<" + value: 700.0 + description: "Decode cap reduced below TDP (700W) under tight budget" + - always: true + field: projected_w + op: "<=" + ref: "planner.total_gpu_power_limit" diff --git a/components/src/dynamo/planner/tests/testbed/scenarios/_base/h100_pcie_disagg.yaml b/components/src/dynamo/planner/tests/testbed/scenarios/_base/h100_pcie_disagg.yaml new file mode 100644 index 000000000000..06f4c90d5d6a --- /dev/null +++ b/components/src/dynamo/planner/tests/testbed/scenarios/_base/h100_pcie_disagg.yaml @@ -0,0 +1,47 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +class: alpha +seed: 42 +ticks: 200 +interval_s: 60 + +planner: + mode: disagg + ttft: 600.0 + itl: 60.0 + enable_power_awareness: true + enable_aic_optimizer: true + total_gpu_power_limit: 2000 + power_agent_safe_default_watts: 280 + prefill_engine_gpu_power_limit: 300 + decode_engine_gpu_power_limit: 250 + aic_initial_c_power_prefill: 1.0 + aic_initial_c_power_decode: 1.0 + aic_initial_c_power_agg: 1.0 + aic_initial_c_ttft: 1.0 + aic_initial_c_itl: 1.0 + aic_reoptimize_interval: 300 + aic_drift_relative_threshold: 0.15 + aic_drift_consecutive_ticks: 3 + aic_max_consecutive_failures: 5 + min_endpoint: 1 + max_gpu_budget: 32 + admission_mode: "off" # MUST be quoted — YAML 1.1 coerces bare `off` to False. + +fleet: + system: h100_pcie + gpus_per_prefill_engine: 1 + gpus_per_decode_engine: 1 + noise: + power_per_gpu: { model: gaussian, sigma: 0.07 } + ttft: { model: gaussian, sigma: 0.05 } + itl: { model: gaussian, sigma: 0.04 } + capacity: { model: gaussian, sigma: 0.03 } + +load: + profile: constant + tokens_per_sec: 800.0 + +events: [] +assertions: [] diff --git a/components/src/dynamo/planner/tests/testbed/scenarios/_base/h200_agg.yaml b/components/src/dynamo/planner/tests/testbed/scenarios/_base/h200_agg.yaml new file mode 100644 index 000000000000..fdd8aff61225 --- /dev/null +++ b/components/src/dynamo/planner/tests/testbed/scenarios/_base/h200_agg.yaml @@ -0,0 +1,47 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +class: alpha +seed: 42 +ticks: 200 +interval_s: 60 + +planner: + mode: agg + ttft: 500.0 + itl: 50.0 + enable_power_awareness: true + enable_aic_optimizer: true + total_gpu_power_limit: 4000 + power_agent_safe_default_watts: 500 + prefill_engine_gpu_power_limit: 490 + decode_engine_gpu_power_limit: 490 + aic_initial_c_power_prefill: 1.0 + aic_initial_c_power_decode: 1.0 + aic_initial_c_power_agg: 1.0 + aic_initial_c_ttft: 1.0 + aic_initial_c_itl: 1.0 + aic_reoptimize_interval: 300 + aic_drift_relative_threshold: 0.15 + aic_drift_consecutive_ticks: 3 + aic_max_consecutive_failures: 5 + min_endpoint: 1 + max_gpu_budget: 64 + admission_mode: "off" # MUST be quoted — YAML 1.1 coerces bare `off` to False. + +fleet: + system: h200_sxm + gpus_per_prefill_engine: 2 + gpus_per_decode_engine: 2 + noise: + power_per_gpu: { model: gaussian, sigma: 0.07 } + ttft: { model: gaussian, sigma: 0.05 } + itl: { model: gaussian, sigma: 0.04 } + capacity: { model: gaussian, sigma: 0.03 } + +load: + profile: constant + tokens_per_sec: 2000.0 + +events: [] +assertions: [] diff --git a/components/src/dynamo/planner/tests/testbed/scenarios/_base/h200_disagg.yaml b/components/src/dynamo/planner/tests/testbed/scenarios/_base/h200_disagg.yaml new file mode 100644 index 000000000000..04efa6fc3b3a --- /dev/null +++ b/components/src/dynamo/planner/tests/testbed/scenarios/_base/h200_disagg.yaml @@ -0,0 +1,53 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +class: alpha +seed: 42 +ticks: 200 +interval_s: 60 + +planner: + mode: disagg + ttft: 500.0 + itl: 50.0 + enable_power_awareness: true + enable_aic_optimizer: true + total_gpu_power_limit: 4000 + power_agent_safe_default_watts: 500 + prefill_engine_gpu_power_limit: 500 + decode_engine_gpu_power_limit: 425 + aic_initial_c_power_prefill: 1.0 + aic_initial_c_power_decode: 1.0 + aic_initial_c_power_agg: 1.0 + aic_initial_c_ttft: 1.0 + aic_initial_c_itl: 1.0 + aic_reoptimize_interval: 300 + aic_drift_relative_threshold: 0.15 + aic_drift_consecutive_ticks: 3 + aic_max_consecutive_failures: 5 + min_endpoint: 1 + max_gpu_budget: 64 + admission_mode: "off" # MUST be quoted — YAML 1.1 coerces bare `off` to False. + +fleet: + system: h200_sxm + gpus_per_prefill_engine: 1 + gpus_per_decode_engine: 2 + bias: + power_bias_prefill: 1.0 + power_bias_decode: 1.0 + ttft_bias: 1.0 + itl_bias: 1.0 + capacity_bias: 1.0 + noise: + power_per_gpu: { model: gaussian, sigma: 0.07 } + ttft: { model: gaussian, sigma: 0.05 } + itl: { model: gaussian, sigma: 0.04 } + capacity: { model: gaussian, sigma: 0.03 } + +load: + profile: constant + tokens_per_sec: 2000.0 + +events: [] +assertions: [] diff --git a/components/src/dynamo/planner/tests/testbed/scenarios/_base/mocker_h200_disagg.yaml b/components/src/dynamo/planner/tests/testbed/scenarios/_base/mocker_h200_disagg.yaml new file mode 100644 index 000000000000..9f8b000cb650 --- /dev/null +++ b/components/src/dynamo/planner/tests/testbed/scenarios/_base/mocker_h200_disagg.yaml @@ -0,0 +1,63 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +class: gamma +seed: 42 +ticks: 60 +interval_s: 60 + +planner: + mode: disagg + ttft: 500.0 + itl: 50.0 + enable_power_awareness: true + enable_aic_optimizer: true + total_gpu_power_limit: 4000 + power_agent_safe_default_watts: 500 + prefill_engine_gpu_power_limit: 500 + decode_engine_gpu_power_limit: 425 + aic_initial_c_power_prefill: 1.0 + aic_initial_c_power_decode: 1.0 + aic_initial_c_power_agg: 1.0 + aic_initial_c_ttft: 1.0 + aic_initial_c_itl: 1.0 + aic_reoptimize_interval: 300 + aic_drift_relative_threshold: 0.15 + aic_drift_consecutive_ticks: 3 + aic_max_consecutive_failures: 5 + min_endpoint: 1 + max_gpu_budget: 64 + admission_mode: "off" # MUST be quoted — YAML 1.1 coerces bare `off` to False. + +mocker: + trace_file: null + synthetic_workload: true + arrival_rate: 200.0 + isl: 3000 + osl: 150 + num_prefill_workers: 1 + num_decode_workers: 4 + router_mode: kv_router + prefill_engine_args: + block_size: 64 + max_num_batched_tokens: 8192 + max_num_seqs: 256 + decode_engine_args: + block_size: 64 + max_num_batched_tokens: 8192 + max_num_seqs: 256 + +overlay: + system: h200_sxm + bias: + power_bias_prefill: 1.0 + power_bias_decode: 1.0 + noise: + power_per_gpu: { model: gaussian, sigma: 0.07 } + +load: + profile: constant + tokens_per_sec: 2000.0 + +events: [] +assertions: [] diff --git a/components/src/dynamo/planner/tests/testbed/scenarios/_base/mocker_h200_synthetic_workload.yaml b/components/src/dynamo/planner/tests/testbed/scenarios/_base/mocker_h200_synthetic_workload.yaml new file mode 100644 index 000000000000..0729e7fe3e83 --- /dev/null +++ b/components/src/dynamo/planner/tests/testbed/scenarios/_base/mocker_h200_synthetic_workload.yaml @@ -0,0 +1,54 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +class: gamma +seed: 42 +ticks: 60 +interval_s: 60 + +planner: + mode: disagg + ttft: 500.0 + itl: 50.0 + enable_power_awareness: true + enable_aic_optimizer: true + total_gpu_power_limit: 4000 + power_agent_safe_default_watts: 500 + prefill_engine_gpu_power_limit: 500 + decode_engine_gpu_power_limit: 425 + aic_initial_c_power_prefill: 1.0 + aic_initial_c_power_decode: 1.0 + aic_reoptimize_interval: 300 + aic_drift_relative_threshold: 0.15 + aic_drift_consecutive_ticks: 3 + aic_max_consecutive_failures: 5 + min_endpoint: 1 + max_gpu_budget: 64 + admission_mode: "off" # MUST be quoted — YAML 1.1 coerces bare `off` to False. + +mocker: + synthetic_workload: true + arrival_rate: 200.0 + isl: 2048 + osl: 512 + num_prefill_workers: 1 + num_decode_workers: 4 + router_mode: kv_router + +overlay: + system: h200_sxm + bias: + power_bias_prefill: 1.0 + power_bias_decode: 1.0 + noise: + power_per_gpu: { model: gaussian, sigma: 0.07 } + +load: + profile: ramp + ramp_from: 200.0 + ramp_to: 2000.0 + ramp_start_tick: 0 + ramp_end_tick: 10 + +events: [] +assertions: [] diff --git a/components/src/dynamo/planner/tests/testbed/synthetic_fleet.py b/components/src/dynamo/planner/tests/testbed/synthetic_fleet.py new file mode 100644 index 000000000000..16d620673a92 --- /dev/null +++ b/components/src/dynamo/planner/tests/testbed/synthetic_fleet.py @@ -0,0 +1,410 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""SyntheticFleet — truth model for α-class scenarios. + +Implements: truth(t) = aic_prediction × bias_signal(t) × (1 + noise(t)) + +Truth-side response to actuation (§4.4): + - cap_p ↓ → TTFT inflates as tdp_w / cap_p (compute-bound) + - cap_d ↓ below decode_power_floor → ITL inflates (memory-bound) + - offered_load > capacity → TTFT/ITL inflate via M/M/1 proxy +""" + +from __future__ import annotations + +import math +import random +from dataclasses import dataclass, field +from typing import TYPE_CHECKING, Optional + +from dynamo.planner.core.types import TrafficObservation + +if TYPE_CHECKING: + from dynamo.planner.tests.testbed.scenarios import ( + BiasRampEvent, + BiasSineEvent, + Event, + FleetSpec, + NoiseModel, + ScenarioSpec, + SystemSpec, + ) + + +@dataclass +class FleetState: + """Mutable fleet state — what's currently running.""" + + n_p_truth: int = 1 + n_d_truth: int = 4 + applied_cap_p: int = 500 + applied_cap_d: int = 425 + + # Bias signals (scenario-controlled) + bias_power_p: float = 1.0 + bias_power_d: float = 1.0 + bias_ttft: float = 1.0 + bias_itl: float = 1.0 + bias_capacity: float = 1.0 + + # AR1 noise state (per signal) + ar1_state: dict[str, float] = field(default_factory=dict) + + +@dataclass +class Observation: + """One tick's truth-side observation (post-noise).""" + + traffic: TrafficObservation + ttft_avg_s: float + itl_avg_s: float + power_w_prefill: float + power_w_decode: float + total_tokens_per_sec: float + + +class SyntheticFleet: + """Truth model for α-class scenarios. + + Computes per-tick observations from AIC predictions multiplied by the + current bias and noise, applying actuation effects (cap clamping, replica + changes) according to fleet state. + """ + + def __init__( + self, + fleet_spec: "FleetSpec", + system_spec: "SystemSpec", + scenario: "ScenarioSpec", + rng: random.Random, + ) -> None: + self._fleet = fleet_spec + self._system = system_spec + self._scenario = scenario + self._rng = rng + + # Initialise state from fleet spec + self.state = FleetState( + n_p_truth=1, + n_d_truth=fleet_spec.gpus_per_decode_engine, + applied_cap_p=scenario.planner.prefill_engine_gpu_power_limit, + applied_cap_d=scenario.planner.decode_engine_gpu_power_limit, + bias_power_p=fleet_spec.bias.power_bias_prefill, + bias_power_d=fleet_spec.bias.power_bias_decode, + bias_ttft=fleet_spec.bias.ttft_bias, + bias_itl=fleet_spec.bias.itl_bias, + bias_capacity=fleet_spec.bias.capacity_bias, + ) + + # Active event tracking + self._active_prom_outage: dict[str, int] = {} # signal → end_tick + self._active_prom_stale: Optional[ + tuple[int, int] + ] = None # (end_tick, lag_ticks) + self._active_actuation_fault: Optional[ + tuple[int, str] + ] = None # (end_tick, mode) + self._active_frontend_fault: Optional[ + tuple[int, float] + ] = None # (end_tick, fraction) + self._active_mdc_unavailable: int = 0 # end_tick + self._sine_events: list[BiasSineEvent] = [] + self._ramp_events: list[BiasRampEvent] = [] + self._window_cross_events: dict[ + str, tuple[int, float] + ] = {} # signal → (tick, weight) + self._observation_history: list[Optional[Observation]] = [] # indexed by tick + + # ------------------------------------------------------------------ + # Main per-tick step + # ------------------------------------------------------------------ + + def step(self, tick: int, offered_load: float) -> Observation: + """Apply pending events, compute the truth-side observation.""" + # Compute AIC predictions at current applied config + aic_ttft_ms = self._system.aic_ttft_ms + aic_itl_ms = self._system.aic_itl_ms + aic_power_w_p = self._system.aic_power_w_prefill + aic_power_w_d = self._system.aic_power_w_decode + + # Compute capacity + base_capacity_tps = ( + self.state.n_d_truth + * self._system.aic_itl_ms + * 10.0 # rough: 10 tok/s per decode replica per GPU at base + ) + # More realistic: seq_per_s × (isl + osl) per replica + # Using AIC ITL: max_concurrency × 1000 / (itl_ms × osl) + max_kv = self._system.max_kv_tokens + osl = 150 # default osl from planner config + isl = 3000 + max_concurrency = max(1, max_kv // (isl + osl)) + seq_per_s_per_replica = max_concurrency * 1000.0 / max(0.001, aic_itl_ms * osl) + base_capacity_tps = seq_per_s_per_replica * self.state.n_d_truth * (isl + osl) + capacity_tps = base_capacity_tps * self.state.bias_capacity + + # Cap effect on compute-bound prefill (TTFT inflates as tdp_w / cap_p) + tdp = self._system.tdp_w + cap_p = max(1, self.state.applied_cap_p) + ttft_cap_factor = tdp / cap_p + true_ttft_ms = aic_ttft_ms * ttft_cap_factor * self.state.bias_ttft + + # Cap effect on memory-bound decode + floor = self._fleet.decode_power_floor_w or self._system.decode_power_floor_w + cap_d = max(1, self.state.applied_cap_d) + if cap_d < floor: + itl_cap_factor = floor / cap_d + else: + itl_cap_factor = 1.0 + true_itl_ms = aic_itl_ms * itl_cap_factor * self.state.bias_itl + + # Queue saturation proxy (M/M/1 inflation when offered > capacity) + utilization = min(0.99, offered_load / max(1.0, capacity_tps)) + queue_factor = 1.0 / max(0.01, 1.0 - utilization) + queue_factor = min(queue_factor, 10.0) # cap to avoid explosion + true_ttft_ms *= queue_factor + true_itl_ms *= queue_factor + + # Power signals + true_power_w_p = aic_power_w_p * self.state.bias_power_p + true_power_w_d = aic_power_w_d * self.state.bias_power_d + + # Apply sine biases + for ev in self._sine_events: + factor = 1.0 + ev.amplitude * math.sin( + 2 * math.pi * tick / max(1, ev.period_ticks) + ev.offset + ) + if ev.signal in ("power_bias_prefill", "power_p"): + true_power_w_p *= factor + elif ev.signal in ("power_bias_decode", "power_d"): + true_power_w_d *= factor + elif ev.signal in ("ttft_bias", "ttft"): + true_ttft_ms *= factor + elif ev.signal in ("itl_bias", "itl"): + true_itl_ms *= factor + + # Apply ramp biases + for ev in self._ramp_events: + if ev.start_tick <= tick <= ev.end_tick: + t = (tick - ev.start_tick) / max(1, ev.end_tick - ev.start_tick) + factor = ev.from_ + t * (ev.to - ev.from_) + _apply_bias_factor(self.state, ev.signal, factor) + + # Apply noise + true_ttft_ms *= 1.0 + self._noise("ttft", tick) + true_itl_ms *= 1.0 + self._noise("itl", tick) + true_power_w_p *= 1.0 + self._noise("power_per_gpu", tick) + true_power_w_d *= 1.0 + self._noise("power_per_gpu", tick) + capacity_tps *= 1.0 + self._noise("capacity", tick) + + # Apply prom window-cross mixing (one-tick mixed window after events) + for signal, (cross_tick, weight_old) in list(self._window_cross_events.items()): + if tick == cross_tick and tick > 0: + prev = ( + self._observation_history[-1] if self._observation_history else None + ) + if prev is not None: + if signal in ("power_d", "power_bias_decode"): + pre_val = prev.power_w_decode + true_power_w_d = ( + weight_old * pre_val + (1 - weight_old) * true_power_w_d + ) + elif signal in ("power_p", "power_bias_prefill"): + pre_val = prev.power_w_prefill + true_power_w_p = ( + weight_old * pre_val + (1 - weight_old) * true_power_w_p + ) + del self._window_cross_events[signal] + + throughput = min(offered_load, max(0.0, capacity_tps)) + traffic = TrafficObservation( + duration_s=self._scenario.interval_s, + num_req=max(0, int(throughput / max(1, isl + osl))), + isl=float(isl), + osl=float(osl), + kv_hit_rate=0.5, + ttft_avg=true_ttft_ms / 1000.0, + itl_avg=true_itl_ms / 1000.0, + total_tokens_per_s=throughput, + scheduled_prefill_tokens=max(0.0, throughput * 0.4), + scheduled_decode_kv_tokens=max(0.0, throughput * 0.6), + ) + + obs = Observation( + traffic=traffic, + ttft_avg_s=true_ttft_ms / 1000.0, + itl_avg_s=true_itl_ms / 1000.0, + power_w_prefill=max(0.0, true_power_w_p), + power_w_decode=max(0.0, true_power_w_d), + total_tokens_per_sec=throughput, + ) + + self._observation_history.append(obs) + return obs + + def observation_at(self, tick: int) -> Optional[Observation]: + """Return observation for given tick (for prom_stale lag).""" + if 0 <= tick < len(self._observation_history): + return self._observation_history[tick] + return None + + # ------------------------------------------------------------------ + # Event application (called by runner per tick) + # ------------------------------------------------------------------ + + def apply_event(self, event: "Event", tick: int) -> None: + """Mutate fleet state based on the event.""" + from dynamo.planner.tests.testbed.scenarios import ( + ActuationFaultEvent, + BiasRampEvent, + BiasSineEvent, + BiasStepEvent, + BudgetChangeEvent, + FrontendPostFaultEvent, + MdcUnavailableEvent, + NodeDownEvent, + NodeUpEvent, + PromOutageEvent, + PromStaleEvent, + PromWindowCrossEvent, + ) + + if isinstance(event, BiasStepEvent): + _apply_bias_factor(self.state, event.signal, event.value) + if event.auto_inject_window_cross: + self._window_cross_events[event.signal] = (tick + 1, 0.66) + + elif isinstance(event, BiasRampEvent): + self._ramp_events.append(event) + + elif isinstance(event, BiasSineEvent): + self._sine_events.append(event) + + elif isinstance(event, ActuationFaultEvent): + self._active_actuation_fault = (tick + event.duration_ticks, event.mode) + + elif isinstance(event, NodeDownEvent): + self.state.n_p_truth = max(0, self.state.n_p_truth - event.n_prefill_lost) + self.state.n_d_truth = max(0, self.state.n_d_truth - event.n_decode_lost) + + elif isinstance(event, NodeUpEvent): + self.state.n_p_truth += event.n_prefill_restored + self.state.n_d_truth += event.n_decode_restored + + elif isinstance(event, PromOutageEvent): + for sig in event.signals: + self._active_prom_outage[sig] = tick + event.duration_ticks + + elif isinstance(event, PromStaleEvent): + self._active_prom_stale = (tick + event.duration_ticks, event.lag_ticks) + + elif isinstance(event, PromWindowCrossEvent): + self._window_cross_events[event.signal] = (tick, event.weight_old) + + elif isinstance(event, BudgetChangeEvent): + # Budget change is handled by the runner (mutates planner config) + pass + + elif isinstance(event, FrontendPostFaultEvent): + self._active_frontend_fault = ( + tick + event.duration_ticks, + event.failing_fraction, + ) + + elif isinstance(event, MdcUnavailableEvent): + self._active_mdc_unavailable = tick + event.duration_ticks + + def clear_expired_events(self, tick: int) -> None: + """Remove expired timed events.""" + expired = [sig for sig, end in self._active_prom_outage.items() if tick >= end] + for sig in expired: + del self._active_prom_outage[sig] + + if self._active_prom_stale and tick >= self._active_prom_stale[0]: + self._active_prom_stale = None + + if self._active_actuation_fault and tick >= self._active_actuation_fault[0]: + self._active_actuation_fault = None + + if self._active_frontend_fault and tick >= self._active_frontend_fault[0]: + self._active_frontend_fault = None + + # ------------------------------------------------------------------ + # Observability helpers for FakePrometheusClient + # ------------------------------------------------------------------ + + def is_signal_in_outage(self, signal: str) -> bool: + return signal in self._active_prom_outage + + def prom_stale_lag(self) -> Optional[int]: + if self._active_prom_stale: + return self._active_prom_stale[1] + return None + + def actuation_fault(self) -> Optional[str]: + if self._active_actuation_fault: + return self._active_actuation_fault[1] + return None + + def frontend_fault(self) -> Optional[float]: + if self._active_frontend_fault: + return self._active_frontend_fault[1] + return None + + def mdc_unavailable(self, tick: int) -> bool: + return tick < self._active_mdc_unavailable + + # ------------------------------------------------------------------ + # Noise + # ------------------------------------------------------------------ + + def _noise(self, signal: str, tick: int) -> float: + noise_spec = getattr(self._fleet.noise, signal, None) + if noise_spec is None: + noise_spec = self._fleet.noise.power_per_gpu + return _sample_noise( + noise_spec, self._rng, f"{signal}_ar1", self.state.ar1_state + ) + + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + + +def _apply_bias_factor(state: FleetState, signal: str, value: float) -> None: + if signal in ("power_bias_prefill", "power_p"): + state.bias_power_p = value + elif signal in ("power_bias_decode", "power_d"): + state.bias_power_d = value + elif signal in ("ttft_bias", "ttft"): + state.bias_ttft = value + elif signal in ("itl_bias", "itl"): + state.bias_itl = value + elif signal in ("capacity_bias", "capacity"): + state.bias_capacity = value + + +def _sample_noise( + spec: "NoiseModel", + rng: random.Random, + ar1_key: str, + ar1_state: dict[str, float], +) -> float: + if spec.model == "gaussian": + sigma = spec.sigma + if sigma == 0.0: + return 0.0 + raw = rng.gauss(0, sigma) + return max(-3 * sigma, min(3 * sigma, raw)) + elif spec.model == "uniform": + h = spec.half_width + return rng.uniform(-h, h) + elif spec.model == "ar1": + prev = ar1_state.get(ar1_key, 0.0) + eps = rng.gauss(0, spec.sigma) + n = spec.rho * prev + eps + ar1_state[ar1_key] = n + return n + return 0.0 diff --git a/components/src/dynamo/planner/tests/testbed/systems/h100_pcie.yaml b/components/src/dynamo/planner/tests/testbed/systems/h100_pcie.yaml new file mode 100644 index 000000000000..14b34317640c --- /dev/null +++ b/components/src/dynamo/planner/tests/testbed/systems/h100_pcie.yaml @@ -0,0 +1,18 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# H100 PCIe — per-SKU constants for the testbed + +tdp_w: 350.0 +sku_min_w: 100.0 +sku_max_w: 350.0 +decode_power_floor_w: 140.0 + +aic_ttft_ms: 180.0 +aic_itl_ms: 12.0 +aic_power_w_prefill: 280.0 +aic_power_w_decode: 220.0 +max_kv_tokens: 100000 + +overlay_prefill_saturation_tokens: 4096 +overlay_decode_hbm_tokens: 100000 diff --git a/components/src/dynamo/planner/tests/testbed/systems/h100_sxm.yaml b/components/src/dynamo/planner/tests/testbed/systems/h100_sxm.yaml new file mode 100644 index 000000000000..79c4d2c54542 --- /dev/null +++ b/components/src/dynamo/planner/tests/testbed/systems/h100_sxm.yaml @@ -0,0 +1,18 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# H100 SXM — per-SKU constants for the testbed + +tdp_w: 700.0 +sku_min_w: 200.0 +sku_max_w: 700.0 +decode_power_floor_w: 280.0 + +aic_ttft_ms: 140.0 +aic_itl_ms: 10.0 +aic_power_w_prefill: 420.0 +aic_power_w_decode: 340.0 +max_kv_tokens: 160000 + +overlay_prefill_saturation_tokens: 8192 +overlay_decode_hbm_tokens: 160000 diff --git a/components/src/dynamo/planner/tests/testbed/systems/h200_sxm.yaml b/components/src/dynamo/planner/tests/testbed/systems/h200_sxm.yaml new file mode 100644 index 000000000000..f9a65ba992ab --- /dev/null +++ b/components/src/dynamo/planner/tests/testbed/systems/h200_sxm.yaml @@ -0,0 +1,21 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# H200 SXM — per-SKU constants for the testbed +# Calibrated from aic_h200_power_data distributions in the repo. + +tdp_w: 700.0 +sku_min_w: 200.0 +sku_max_w: 700.0 +decode_power_floor_w: 280.0 + +# FakeAIC estimator responses (§8.4) +aic_ttft_ms: 120.0 +aic_itl_ms: 8.0 +aic_power_w_prefill: 450.0 +aic_power_w_decode: 360.0 +max_kv_tokens: 200000 + +# γ-class SyntheticPowerOverlay formula constants (§5.2) +overlay_prefill_saturation_tokens: 8192 +overlay_decode_hbm_tokens: 200000 diff --git a/components/src/dynamo/planner/tests/testbed/test_scenarios.py b/components/src/dynamo/planner/tests/testbed/test_scenarios.py new file mode 100644 index 000000000000..159cfacd7cf5 --- /dev/null +++ b/components/src/dynamo/planner/tests/testbed/test_scenarios.py @@ -0,0 +1,110 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +""" +pytest entry point: parametrize over every scenario YAML in scenarios/. + +Usage: + # run all scenarios + pytest components/src/dynamo/planner/tests/testbed/test_scenarios.py -v + + # run only α-class scenarios + pytest components/src/dynamo/planner/tests/testbed/test_scenarios.py -v \ + -k "alpha" + + # run only γ-class scenarios (requires mocker trace) + pytest components/src/dynamo/planner/tests/testbed/test_scenarios.py -v \ + -k "gamma" +""" +from __future__ import annotations + +from pathlib import Path + +import pytest + +pytestmark = [ + pytest.mark.gpu_0, + pytest.mark.pre_merge, + pytest.mark.integration, + pytest.mark.planner, +] + +# --------------------------------------------------------------------------- +# Scenario discovery +# --------------------------------------------------------------------------- +_SCENARIOS_DIR = Path(__file__).parent / "scenarios" +_ALPHA_YAMLS = sorted( + p + for p in _SCENARIOS_DIR.glob("*.yaml") + if not p.stem.startswith("_") and not p.stem.startswith("G") +) +_GAMMA_YAMLS = sorted(p for p in _SCENARIOS_DIR.glob("G*.yaml")) + + +def _scenario_id(path: Path) -> str: + return path.stem + + +# --------------------------------------------------------------------------- +# α-class +# --------------------------------------------------------------------------- +@pytest.mark.testbed +@pytest.mark.parametrize( + "scenario_path", + _ALPHA_YAMLS, + ids=[_scenario_id(p) for p in _ALPHA_YAMLS], +) +def test_alpha(scenario_path: Path, tmp_path: Path) -> None: + """Run one α-class scenario and assert all expectations pass.""" + from dynamo.planner.tests.testbed.assertions import evaluate_all + from dynamo.planner.tests.testbed.runner import ScenarioRunner + from dynamo.planner.tests.testbed.scenarios import load_scenario + + scenario = load_scenario(scenario_path) + runner = ScenarioRunner(scenario) + history = runner.run() + + csv_path = tmp_path / f"{scenario.name}.csv" + history.to_csv(csv_path) + + failures = evaluate_all(history, scenario) + if failures: + msg = "\n".join(f" [{i}] {f}" for i, f in enumerate(failures, 1)) + pytest.fail( + f"Scenario {scenario.name!r}: {len(failures)} assertion(s) failed:\n{msg}" + ) + + +# --------------------------------------------------------------------------- +# γ-class +# --------------------------------------------------------------------------- +@pytest.mark.testbed +@pytest.mark.gamma +@pytest.mark.parametrize( + "scenario_path", + _GAMMA_YAMLS, + ids=[_scenario_id(p) for p in _GAMMA_YAMLS], +) +def test_gamma(scenario_path: Path, tmp_path: Path) -> None: + """Run one γ-class scenario and assert all expectations pass.""" + pytest.importorskip( + "dynamo.llm", + reason="γ-class scenarios require the dynamo.llm (mocker) package", + ) + from dynamo.planner.tests.testbed.assertions import evaluate_all + from dynamo.planner.tests.testbed.runner import ScenarioRunner + from dynamo.planner.tests.testbed.scenarios import load_scenario + + scenario = load_scenario(scenario_path) + runner = ScenarioRunner(scenario) + history = runner.run() + + csv_path = tmp_path / f"{scenario.name}.csv" + history.to_csv(csv_path) + + failures = evaluate_all(history, scenario) + if failures: + msg = "\n".join(f" [{i}] {f}" for i, f in enumerate(failures, 1)) + pytest.fail( + f"Scenario {scenario.name!r}: {len(failures)} assertion(s) failed:\n{msg}" + ) diff --git a/components/src/dynamo/planner/tests/testbed/tests/__init__.py b/components/src/dynamo/planner/tests/testbed/tests/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/components/src/dynamo/planner/tests/testbed/tests/test_aic_real_data.py b/components/src/dynamo/planner/tests/testbed/tests/test_aic_real_data.py new file mode 100644 index 000000000000..44af4aeab0a1 --- /dev/null +++ b/components/src/dynamo/planner/tests/testbed/tests/test_aic_real_data.py @@ -0,0 +1,508 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""End-to-end tests against a *real* AIC perf database. + +These tests are opt-in: they run only when the environment variable +``AIC_SANDBOX_DIR`` points at a populated ``systems/`` directory containing +both the AIC system YAMLs (e.g. ``h200_sxm.yaml``) and a ``data/`` tree +matching the in-repo layout used by ``aiconfigurator.sdk.perf_database``. +On CI you typically mount a read-only copy of the AIC power-data tarball +and point ``AIC_SANDBOX_DIR`` at it. + +What we verify against the live data: + +1. ``AIConfiguratorPerfEstimator`` loads cleanly and reports non-zero + ``power_w`` for both prefill and decode on a representative workload. +2. ``AICPowerOptimizer.optimize()`` produces a ``PowerAwareConfig`` with + per-GPU caps that never exceed nameplate TDP × 1.1 (the defensive clamp + from §8 row 14 of ``powerplanner-design.md``). +3. The clamp counter ``aic_power_w_clamped_total{side=...}`` fires iff + AIC's raw ``power_w`` for that side was outside the physical envelope. +4. The multi-tick EMA loop converges in three regimes (well-calibrated, + AIC-over-predicts → pegs at 0.5, AIC-under-predicts → 1.67) and + ``should_reoptimize()`` respects the hysteresis count. + +Why this lives in the testbed rather than in +``integration/test_aic_power_optimizer.py``: + +* It needs ``aiconfigurator`` installed and a real sandbox on disk — a + much heavier prerequisite than the rest of the integration suite, which + uses a pure ``MagicMock`` estimator. +* It belongs alongside the other "real-system bridge" tests: the testbed + is the dedicated home for "exercise the planner against real-ish data + without a Kubernetes cluster". +* Gating via env var keeps the default test invocation cheap; CI opts in + by setting ``AIC_SANDBOX_DIR`` once. +""" +from __future__ import annotations + +import logging +import os +import random +import time +from pathlib import Path +from typing import Iterator +from unittest.mock import MagicMock + +import pytest + +# --------------------------------------------------------------------------- +# Hard gate: only collect when AIC_SANDBOX_DIR is set AND aiconfigurator is +# importable. Both checks fail-skip the entire module rather than the +# individual tests so the skip reason shows up exactly once. +# --------------------------------------------------------------------------- + +_SANDBOX_ENV = os.environ.get("AIC_SANDBOX_DIR") +if not _SANDBOX_ENV: + pytest.skip( + "AIC_SANDBOX_DIR not set — opt-in test, see " + "docs/design-docs/powerplanner-testbed-design.md for sandbox setup. " + "Locally: AIC_SANDBOX_DIR=/.aic_sandbox/systems pytest -m real_aic ...", + allow_module_level=True, + ) + +_SANDBOX_PATH = Path(_SANDBOX_ENV) +if not _SANDBOX_PATH.is_dir(): + pytest.skip( + f"AIC_SANDBOX_DIR={_SANDBOX_ENV} is not a directory.", + allow_module_level=True, + ) + +# Probe the AIC package up front — without it, none of this can run. +pytest.importorskip( + "aiconfigurator.sdk.perf_database", + reason="real-AIC tests require the aiconfigurator package; " + "`pip install aiconfigurator` or run inside the dev pod.", +) + +from dynamo.planner.config.aic_interpolation_spec import ( # noqa: E402 + AICInterpolationSpec, +) +from dynamo.planner.config.parallelization import PickedParallelConfig # noqa: E402 +from dynamo.planner.config.planner_config import PlannerConfig # noqa: E402 +from dynamo.planner.core.types import TrafficObservation # noqa: E402 +from dynamo.planner.monitoring.aic_power_optimizer import ( # noqa: E402 + AICPowerOptimizer, +) + +logger = logging.getLogger(__name__) + + +# --------------------------------------------------------------------------- +# Sandbox plumbing — re-route AIC's path resolver at session scope. +# --------------------------------------------------------------------------- + + +@pytest.fixture(scope="session", autouse=True) +def _patch_aic_systems_dir() -> Iterator[None]: + """Point AIC's perf-database loader at AIC_SANDBOX_DIR for the whole session. + + AIC computes its systems path at function-definition time (the + ``systems_dir`` parameter has ``get_system_config_path()`` baked into + its ``__defaults__``), so we must rewrite three functions' defaults + in-place AND override the resolver itself. Yes, this is ugly; no, the + alternative (forking AIC) is uglier. See aic_smoke.py in the repo + root for the original isolation of this technique. + """ + import aiconfigurator.sdk.perf_database as pd + + sandbox = str(_SANDBOX_PATH) + orig_resolver = pd.get_system_config_path + orig_defaults: dict[str, tuple] = {} + + def _override_resolver() -> str: + return sandbox + + pd.get_system_config_path = _override_resolver + + for fn_name in ("get_supported_databases", "get_database", "get_all_databases"): + fn = getattr(pd, fn_name, None) + if fn is None or fn.__defaults__ is None: + continue + orig_defaults[fn_name] = fn.__defaults__ + n_pos = fn.__code__.co_argcount + arg_names = fn.__code__.co_varnames[:n_pos] + defaults = list(fn.__defaults__) + first_default_idx = n_pos - len(defaults) + for i, name in enumerate(arg_names[first_default_idx:]): + if name == "systems_dir": + defaults[i] = sandbox + fn.__defaults__ = tuple(defaults) + + pd.databases_cache.clear() + try: + yield + finally: + pd.get_system_config_path = orig_resolver + for fn_name, defaults in orig_defaults.items(): + fn = getattr(pd, fn_name) + fn.__defaults__ = defaults + pd.databases_cache.clear() + + +# --------------------------------------------------------------------------- +# Per-system parametrization — drives every test against every (system, +# backend, hf_id) tuple that is actually present in the sandbox. +# --------------------------------------------------------------------------- + + +def _discover_systems() -> list[dict]: + """Return the list of (system, backend, hf_id, tdp_w_expected) combos to test. + + Each entry is checked against the live sandbox at collection time so + missing data on the sandbox skips that specific parametrization rather + than failing it. + """ + import aiconfigurator.sdk.perf_database as pd + + out: list[dict] = [] + candidates = [ + # (system, backend, hf_id, expected_tdp_w, expected_clamp_decode) + ("h200_sxm", "vllm", "LLAMA3.1_8B", 700.0, True), # known to clamp (1275 W) + ("b200_sxm", "trtllm", "LLAMA3.1_8B", 1000.0, False), # stays in envelope + ] + available = pd.get_supported_databases() + for sys_name, backend, hf_id, tdp, clamp_expected in candidates: + backends = available.get(sys_name, {}) + versions = backends.get(backend, []) + if not versions: + continue + out.append( + { + "system": sys_name, + "backend": backend, + "hf_id": hf_id, + "tdp_w": tdp, + "expect_decode_clamp": clamp_expected, + "latest_version": sorted(versions)[-1], + } + ) + return out + + +_SKU_TABLE = _discover_systems() + + +def _sku_id(sku: dict) -> str: + return f"{sku['system']}-{sku['backend']}-{sku['hf_id']}" + + +# --------------------------------------------------------------------------- +# Helper factories +# --------------------------------------------------------------------------- + + +def _make_config( + *, system: str, backend: str, hf_id: str, isl: int, osl: int +) -> PlannerConfig: + interp = AICInterpolationSpec( + hf_id=hf_id, + system=system, + backend=backend, + isl=isl, + osl=osl, + sweep_max_context_length=isl + osl, + prefill_interpolation_granularity=8, + decode_interpolation_granularity=8, + prefill_pick=PickedParallelConfig(tp=1, pp=1, dp=1, moe_tp=1, moe_ep=1), + decode_pick=PickedParallelConfig(tp=1, pp=1, dp=1, moe_tp=1, moe_ep=1), + ) + return PlannerConfig( + namespace="real-aic-testbed", + environment="virtual", + backend=backend, + mode="disagg", + enable_aic_optimizer=True, + aic_interpolation=interp, + aic_system=system, + enable_power_awareness=True, + total_gpu_power_limit=16000, + prefill_engine_gpu_power_limit=int(1500), + decode_engine_gpu_power_limit=int(1500), + power_agent_safe_default_watts=500, + min_endpoint=1, + max_gpu_budget=8, + ttft=10_000.0, + itl=10_000.0, + ) + + +# --------------------------------------------------------------------------- +# Tests +# --------------------------------------------------------------------------- + + +@pytest.mark.real_aic +@pytest.mark.parametrize( + "sku", _SKU_TABLE, ids=[_sku_id(s) for s in _SKU_TABLE] or ["no-skus-found"] +) +class TestAICRealData: + """End-to-end checks against the real AIC perf database. + + The class-level parametrize lets a single sandbox cover multiple SKUs + in a single pytest run. + """ + + ISL, OSL = 2048, 256 + + def test_estimator_returns_real_power_w(self, sku: dict) -> None: + """Direct estimator must produce non-zero power_w.""" + if not _SKU_TABLE: + pytest.skip("no SKUs available in sandbox") + from dynamo.planner.monitoring.aic_estimator import AIConfiguratorPerfEstimator + + est = AIConfiguratorPerfEstimator( + hf_id=sku["hf_id"], system=sku["system"], backend=sku["backend"] + ) + assert float(est.database.system_spec["gpu"]["power"]) == pytest.approx( + sku["tdp_w"] + ) + + prefill = est.estimate_perf(self.ISL, self.OSL, 8, mode="prefill", tp_size=1) + decode = est.estimate_perf(self.ISL, self.OSL, 8, mode="decode", tp_size=1) + prefill_w = float(prefill.get("power_w") or 0.0) + decode_w = float(decode.get("power_w") or 0.0) + assert prefill_w > 0, ( + f"{_sku_id(sku)}: AIC returned power_w=0 for prefill — " + "power data not actually loaded in sandbox" + ) + assert decode_w > 0 + # Per-op direct call at batch=8 should NEVER extrapolate non-physical. + assert prefill_w <= sku["tdp_w"] * 1.1, ( + f"direct estimate exceeded TDP×1.1 at small batch — " + f"sandbox data may be corrupt for {_sku_id(sku)}" + ) + assert decode_w <= sku["tdp_w"] * 1.1 + + def test_optimize_produces_well_formed_config(self, sku: dict) -> None: + cfg = _make_config( + system=sku["system"], + backend=sku["backend"], + hf_id=sku["hf_id"], + isl=self.ISL, + osl=self.OSL, + ) + opt = AICPowerOptimizer(config=cfg, metrics=MagicMock()) + result = opt.optimize() + + assert result is not None, "optimize() unexpectedly returned None" + assert result.n_p >= 1 and result.n_d >= 1 + assert result.cap_p > 0 and result.cap_d > 0 + # Caps must NEVER exceed nameplate TDP × _COEFF_MAX (2.0) when c_power + # coefficient has drifted up; that's fine, the underlying *aic_power_w* + # is what must be bounded, and that's what the clamp guards. + assert result.cap_d <= sku["tdp_w"] * 2.0 + 1 + assert result.cap_p <= sku["tdp_w"] * 2.0 + 1 + + def test_clamp_engages_as_expected(self, sku: dict) -> None: + """Clamp counter fires iff raw aic_power_w > 1.1 × TDP.""" + cfg = _make_config( + system=sku["system"], + backend=sku["backend"], + hf_id=sku["hf_id"], + isl=self.ISL, + osl=self.OSL, + ) + metrics = MagicMock() + opt = AICPowerOptimizer(config=cfg, metrics=metrics) + result = opt.optimize() + assert result is not None + + threshold = sku["tdp_w"] * 1.1 + expected_clamped: list[str] = [] + if result.aic_power_w_prefill > threshold: + expected_clamped.append("prefill") + if result.aic_power_w_decode > threshold: + expected_clamped.append("decode") + + sides = sorted( + c.kwargs.get("side") + for c in metrics.aic_power_w_clamped_total.labels.call_args_list + ) + assert sides == sorted(expected_clamped), ( + f"{_sku_id(sku)}: clamp sides mismatch — got {sides}, " + f"expected {sorted(expected_clamped)} " + f"(raw prefill={result.aic_power_w_prefill:.1f} W, " + f"raw decode={result.aic_power_w_decode:.1f} W, " + f"threshold={threshold:.0f} W)" + ) + + # For the known H200 vLLM 1275 W case, also assert the *applied* cap + # came out at or below TDP — that's the user-visible contract. + if sku["expect_decode_clamp"]: + assert result.aic_power_w_decode > threshold, ( + f"sandbox H200 should still exhibit the 1275 W extrapolation; " + f"got {result.aic_power_w_decode:.1f} W instead — has the " + f"data been re-collected?" + ) + assert result.cap_d <= sku["tdp_w"] + 1, ( + f"cap_d={result.cap_d} W must be clamped to ~TDP " + f"({sku['tdp_w']:.0f} W) when AIC over-predicts" + ) + + +@pytest.mark.real_aic +class TestAICDriftLoopRealData: + """Multi-tick EMA loop against the real H200 sandbox. + + Drives update_correction() with synthetic observation streams whose + means are anchored to physical (NOT AIC) values; the AIC denominators + come from a real optimize() call. This is the closest thing to a + production loop we can run without a real GPU. + """ + + ISL, OSL = 2048, 256 + HF_ID = "LLAMA3.1_8B" + + @pytest.fixture + def h200_optimizer(self) -> AICPowerOptimizer: + """Skip unless the H200 SXM vLLM data is in the sandbox.""" + import aiconfigurator.sdk.perf_database as pd + + available = pd.get_supported_databases().get("h200_sxm", {}).get("vllm", []) + if not available: + pytest.skip("H200 SXM + vLLM data not in sandbox") + cfg = _make_config( + system="h200_sxm", + backend="vllm", + hf_id=self.HF_ID, + isl=self.ISL, + osl=self.OSL, + ) + opt = AICPowerOptimizer(config=cfg, metrics=MagicMock()) + result = opt.optimize() + assert result is not None + return opt + + @staticmethod + def _drive( + opt: AICPowerOptimizer, + *, + observed_power_w_decode_mean: float, + n_ticks: int = 80, + noise_frac: float = 0.05, + seed: int = 42, + ) -> float: + rng = random.Random(seed) + for _ in range(n_ticks): + traffic = TrafficObservation( + duration_s=60.0, + num_req=10.0, + isl=2048.0, + osl=256.0, + ttft_avg=0.05, + itl_avg=0.01, + total_tokens_per_s=200.0, + scheduled_prefill_tokens=2000.0, + scheduled_decode_kv_tokens=2000.0, + ) + obs = observed_power_w_decode_mean * ( + 1.0 + rng.uniform(-noise_frac, noise_frac) + ) + opt.update_correction( + traffic=traffic, + observed_ttft_avg=traffic.ttft_avg, + observed_itl_avg=traffic.itl_avg, + observed_power_w_prefill=obs * 0.6, + observed_power_w_decode=obs, + ) + return opt._c_power_d + + def test_well_calibrated_converges_to_one( + self, h200_optimizer: AICPowerOptimizer + ) -> None: + """Observed == aic_power_w_decode → c_power_d converges to 1.0.""" + target = h200_optimizer._last_optimal_config.aic_power_w_decode + final = self._drive(h200_optimizer, observed_power_w_decode_mean=target) + assert ( + abs(final - 1.0) < 0.05 + ), f"c_power_d={final:.3f} did not converge to 1.0 under matched observations" + + def test_over_prediction_pegs_at_lower_clamp( + self, h200_optimizer: AICPowerOptimizer + ) -> None: + """Observed = 500 W < raw aic 1275 W → c_power_d hits 0.5 clamp.""" + raw_aic_decode = h200_optimizer._last_optimal_config.aic_power_w_decode + if raw_aic_decode < 1000: + pytest.skip( + f"H200 sandbox no longer exhibits the >1000 W decode artefact " + f"(raw={raw_aic_decode:.0f} W); drift-peg test is moot." + ) + final = self._drive(h200_optimizer, observed_power_w_decode_mean=500.0) + assert final == pytest.approx( + 0.5 + ), f"c_power_d={final:.3f} should peg at 0.5 when observed << aic" + + def test_under_prediction_converges_below_upper_clamp( + self, h200_optimizer: AICPowerOptimizer + ) -> None: + """Observed > raw_aic but ratio < 2 → c_power_d converges within (1, 2).""" + raw = h200_optimizer._last_optimal_config.aic_power_w_decode + # Pick an observed value that gives ratio ≈ 1.5 (well inside the clamp). + target_ratio = 1.5 + observed = raw * target_ratio + final = self._drive(h200_optimizer, observed_power_w_decode_mean=observed) + # Allow a small noise margin; ratio derived from noisy observations + # may differ from target_ratio by ~the noise frac. + assert 1.4 < final < 1.6, ( + f"c_power_d={final:.3f} did not converge to ~{target_ratio} " + f"(raw aic_decode={raw:.0f} W, observed mean={observed:.0f} W)" + ) + + def test_hysteresis_holds_on_sla_drift( + self, h200_optimizer: AICPowerOptimizer + ) -> None: + """should_reoptimize fires only after aic_drift_consecutive_ticks.""" + opt = h200_optimizer + opt._time_of_last_optimize = ( + time.monotonic() - opt._config.aic_reoptimize_interval - 1.0 + ) + hysteresis = opt._config.aic_drift_consecutive_ticks + triggered_at: int | None = None + for tick in range(hysteresis + 3): + traffic = TrafficObservation( + duration_s=60.0, + num_req=10.0, + isl=2048.0, + osl=256.0, + ttft_avg=opt._config.ttft_ms / 1000.0 * 2.0, # 2× SLA + itl_avg=0.005, + total_tokens_per_s=200.0, + scheduled_prefill_tokens=2000.0, + scheduled_decode_kv_tokens=2000.0, + ) + if opt.should_reoptimize(traffic): + triggered_at = tick + break + assert ( + triggered_at is not None + ), "should_reoptimize never fired under sustained SLA breach" + assert triggered_at + 1 == hysteresis, ( + f"hysteresis broken: triggered at tick={triggered_at}, " + f"expected exactly tick={hysteresis - 1}" + ) + + def test_healthy_load_does_not_trigger_reoptimize( + self, h200_optimizer: AICPowerOptimizer + ) -> None: + opt = h200_optimizer + opt._time_of_last_optimize = ( + time.monotonic() - opt._config.aic_reoptimize_interval - 1.0 + ) + opt._estimated_throughput = 1000.0 + for tick in range(50): + traffic = TrafficObservation( + duration_s=60.0, + num_req=10.0, + isl=2048.0, + osl=256.0, + ttft_avg=0.020, # well under SLA + itl_avg=0.005, + total_tokens_per_s=500.0, + scheduled_prefill_tokens=2000.0, + scheduled_decode_kv_tokens=2000.0, + ) + assert not opt.should_reoptimize( + traffic + ), f"spurious reoptimize at tick {tick} under healthy load" diff --git a/components/src/dynamo/planner/tests/testbed/tests/test_fakes.py b/components/src/dynamo/planner/tests/testbed/tests/test_fakes.py new file mode 100644 index 000000000000..ac2c92a1263f --- /dev/null +++ b/components/src/dynamo/planner/tests/testbed/tests/test_fakes.py @@ -0,0 +1,284 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Unit tests for the testbed's fake components. + +These tests are deliberately constructed against the current public API of +each fake — if you rename a field on TickSnapshot, change FakeAIC's seam, or +restructure FakeActuator, these tests fail loud at collection. +""" +from __future__ import annotations + +import random +from typing import TYPE_CHECKING + +import pytest + +if TYPE_CHECKING: + from dynamo.planner.tests.testbed.scenarios import ScenarioSpec + +pytestmark = [ + pytest.mark.gpu_0, + pytest.mark.pre_merge, + pytest.mark.unit, + pytest.mark.planner, +] + +# --------------------------------------------------------------------------- +# Shared helpers +# --------------------------------------------------------------------------- + + +def _system_spec(): + from dynamo.planner.tests.testbed.scenarios import SystemSpec + + return SystemSpec.load("h200_sxm") + + +def _scenario( + planner_overrides: dict | None = None, fleet_overrides: dict | None = None +) -> "ScenarioSpec": # type: ignore[name-defined] + from dynamo.planner.tests.testbed.scenarios import ( + FleetSpec, + LoadSpec, + PlannerSpec, + ScenarioSpec, + ) + + planner_kwargs = dict( + mode="disagg", + enable_power_awareness=True, + enable_aic_optimizer=True, + total_gpu_power_limit=4000, + power_agent_safe_default_watts=500, + prefill_engine_gpu_power_limit=500, + decode_engine_gpu_power_limit=425, + ) + if planner_overrides: + planner_kwargs.update(planner_overrides) + + fleet_kwargs = dict(system="h200_sxm") + if fleet_overrides: + fleet_kwargs.update(fleet_overrides) + + return ScenarioSpec( + name="test", + **{"class": "alpha"}, + ticks=10, + planner=PlannerSpec(**planner_kwargs), + fleet=FleetSpec(**fleet_kwargs), + load=LoadSpec(profile="constant", tokens_per_sec=1000.0), + ) + + +def _fleet(scenario=None): + from dynamo.planner.tests.testbed.synthetic_fleet import SyntheticFleet + + sc = scenario or _scenario() + return SyntheticFleet(sc.fleet, _system_spec(), sc, random.Random(0)) + + +def _metrics(): + from dynamo.planner.tests.testbed.fake_planner_metrics import FakePlannerMetrics + + return FakePlannerMetrics() + + +# --------------------------------------------------------------------------- +# FakeAIC +# --------------------------------------------------------------------------- + + +class TestFakeAIC: + def test_factory_returns_estimator_with_normal_values(self): + from dynamo.planner.tests.testbed.fake_aic import FakeAIC + + aic = FakeAIC(_system_spec()) + factory = aic.make_estimator_factory() + est = factory(hf_id="m", system="h200_sxm", backend="vllm") + + prefill = est.estimate_prefill_perf(isl=3000) + decode = est.estimate_perf(isl=3000, osl=150, batch_size=1) + + # Values come from systems/h200_sxm.yaml — keep test loosely coupled + # to the actual numbers but assert plausibility. + assert prefill["context_latency"] > 0 + assert prefill["power_w"] > 0 + assert decode["tpot"] > 0 + assert decode["power_w"] > 0 + + def test_raises_mode(self): + from dynamo.planner.tests.testbed.fake_aic import FakeAIC + + aic = FakeAIC(_system_spec()) + aic.set_fault_mode("raises") + est = aic.make_estimator_factory()(hf_id="m", system="s", backend="vllm") + with pytest.raises(RuntimeError): + est.estimate_prefill_perf(isl=3000) + + def test_empty_pareto_mode_returns_huge_ttft(self): + """``empty_pareto`` forces the optimizer into the infeasibility path.""" + from dynamo.planner.tests.testbed.fake_aic import FakeAIC + + aic = FakeAIC(_system_spec()) + aic.set_fault_mode("empty_pareto") + est = aic.make_estimator_factory()(hf_id="m", system="s", backend="vllm") + prefill = est.estimate_prefill_perf(isl=3000) + assert prefill["context_latency"] > 100_000 + + def test_reset_fault_restores_normal_response(self): + from dynamo.planner.tests.testbed.fake_aic import FakeAIC + + aic = FakeAIC(_system_spec()) + aic.set_fault_mode("raises") + aic.reset_fault() + est = aic.make_estimator_factory()(hf_id="m", system="s", backend="vllm") + # Should not raise now. + est.estimate_prefill_perf(isl=3000) + + +# --------------------------------------------------------------------------- +# FakePlannerMetrics +# --------------------------------------------------------------------------- + + +class TestFakePlannerMetrics: + def test_counter_starts_zero_and_increments(self): + m = _metrics() + assert m.aic_optimizer_exceptions_total.value == 0.0 + m.aic_optimizer_exceptions_total.inc() + m.aic_optimizer_exceptions_total.inc() + assert m.aic_optimizer_exceptions_total.value == 2.0 + + def test_labeled_counter(self): + m = _metrics() + m.power_agent_cap_clamped_total.labels(direction="min").inc() + m.power_agent_cap_clamped_total.labels(direction="min").inc() + m.power_agent_cap_clamped_total.labels(direction="max").inc() + assert m.power_agent_cap_clamped_total.labeled_value(direction="min") == 2.0 + assert m.power_agent_cap_clamped_total.labeled_value(direction="max") == 1.0 + + def test_gauge_set(self): + m = _metrics() + m.aic_consecutive_failures.set(3) + assert m.aic_consecutive_failures.value == 3 + + +# --------------------------------------------------------------------------- +# FakeActuator +# --------------------------------------------------------------------------- + + +class TestFakeActuator: + def _make_actuator(self, scenario=None): + from dynamo.planner.tests.testbed.fake_actuator import FakeActuator + + sc = scenario or _scenario() + fleet = _fleet(sc) + actuator = FakeActuator(sc, fleet, _metrics(), _system_spec()) + return actuator, fleet + + def test_apply_caps_within_sku_range(self): + actuator, _ = self._make_actuator() + actuator.apply_caps(500, 425) + snap = actuator.applied_caps_snapshot() + sys = _system_spec() + assert sys.sku_min_w <= snap.cap_p <= sys.sku_max_w + assert sys.sku_min_w <= snap.cap_d <= sys.sku_max_w + + def test_apply_caps_clamps_below_min(self): + actuator, _ = self._make_actuator() + actuator.apply_caps(1, 1) + snap = actuator.applied_caps_snapshot() + sys = _system_spec() + assert snap.cap_p >= sys.sku_min_w + assert snap.cap_d >= sys.sku_min_w + + def test_apply_caps_clamps_above_max(self): + actuator, _ = self._make_actuator() + actuator.apply_caps(9999, 9999) + snap = actuator.applied_caps_snapshot() + sys = _system_spec() + assert snap.cap_p <= sys.sku_max_w + assert snap.cap_d <= sys.sku_max_w + + def test_rbac_denied_blocks_apply_replicas(self): + actuator, _ = self._make_actuator() + actuator.set_actuation_fault("rbac_denied") + with pytest.raises(RuntimeError, match="403"): + actuator.apply_replicas(2, 4) + + def test_apply_replicas_propagates_to_fleet_state(self): + actuator, fleet = self._make_actuator() + actuator.apply_replicas(3, 5) + assert fleet.state.n_p_truth == 3 + assert fleet.state.n_d_truth == 5 + + def test_nvml_low_clamp_records_metric(self): + actuator, _ = self._make_actuator() + actuator.set_actuation_fault("nvml_low") + actuator.apply_caps(50, 50) # Below sku_min_w + snap = actuator.applied_caps_snapshot() + sys = _system_spec() + assert snap.cap_p >= sys.sku_min_w + assert snap.cap_d >= sys.sku_min_w + + +# --------------------------------------------------------------------------- +# FakePrometheusClient +# --------------------------------------------------------------------------- + + +class TestFakePrometheusClient: + def _make_prom(self): + from dynamo.planner.tests.testbed.fake_prometheus import FakePrometheusClient + + sc = _scenario() + fleet = _fleet(sc) + prom = FakePrometheusClient(source=fleet) + prom.set_tick(0) + # Drive a tick so observation_at(0) has data. + fleet.step(tick=0, offered_load=1000.0) + return prom, fleet + + def test_decode_power_is_positive_float(self): + prom, _ = self._make_prom() + val = prom.get_avg_per_gpu_power_by_component( + component="decode", interval="60s" + ) + assert isinstance(val, float) + assert val > 0 + + def test_prefill_power_is_positive_float(self): + prom, _ = self._make_prom() + val = prom.get_avg_per_gpu_power_by_component( + component="prefill", interval="60s" + ) + assert isinstance(val, float) + assert val > 0 + + def test_outage_returns_none(self): + prom, fleet = self._make_prom() + # Inject a power_p outage active at tick 0. + fleet._active_prom_outage["power_p"] = 10 + val = prom.get_avg_per_gpu_power_by_component( + component="prefill", interval="60s" + ) + assert val is None + + +# --------------------------------------------------------------------------- +# Wire-up sanity: scenarios sourced from real YAMLs load via the same code path +# --------------------------------------------------------------------------- + + +def test_load_a1_scenario_smoke(): + from pathlib import Path + + from dynamo.planner.tests.testbed.scenarios import load_scenario + + here = Path(__file__).parent.parent / "scenarios" + sc = load_scenario(here / "A1_power_under_estimate_decode.yaml") + assert sc.class_name == "alpha" + assert sc.fleet is not None + assert sc.fleet.bias.power_bias_decode == 1.35 diff --git a/components/src/dynamo/planner/tests/testbed/tests/test_overlay.py b/components/src/dynamo/planner/tests/testbed/tests/test_overlay.py new file mode 100644 index 000000000000..dee83fa2c15e --- /dev/null +++ b/components/src/dynamo/planner/tests/testbed/tests/test_overlay.py @@ -0,0 +1,180 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Tests for SyntheticPowerOverlay (γ-class). + +Validates the in-process power synthesiser without requiring the Rust +mocker binding. Covers: + 1. Determinism: same RNG seed → byte-identical observation + 2. Determinism: different seeds with non-zero noise → diverge + 3. Bias propagation: ``power_bias_decode`` scales decode power 1:1 + 4. Bias independence: decode bias doesn't perturb prefill power + 5. Bias propagation: ``power_bias_prefill`` scales prefill power 1:1 + 6. Zero-bias sanity: output stays within reasonable [base, cap_w] range +""" +from __future__ import annotations + +import math +import random + +import pytest + +pytestmark = [ + pytest.mark.gpu_0, + pytest.mark.pre_merge, + pytest.mark.unit, + pytest.mark.planner, +] + +# --------------------------------------------------------------------------- +# Helpers — build an overlay against the actual current API +# (SystemSpec is per-SKU constants; OverlaySpec.system is a SKU name string). +# --------------------------------------------------------------------------- + + +def _make_scenario(bias_decode: float = 1.0, bias_prefill: float = 1.0): + """Construct a minimal γ scenario spec used only to drive the overlay.""" + from dynamo.planner.tests.testbed.scenarios import ( + BiasSpec, + FleetSpec, + LoadSpec, + MockerSpec, + OverlaySpec, + PlannerSpec, + ScenarioSpec, + ) + + return ScenarioSpec( + name="overlay-unit", + class_="gamma", + description="", + seed=42, + ticks=10, + interval_s=60, + planner=PlannerSpec( + prefill_engine_gpu_power_limit=450, + decode_engine_gpu_power_limit=360, + ), + fleet=FleetSpec(), + mocker=MockerSpec(synthetic_workload=True), + overlay=OverlaySpec( + system="h200_sxm", + bias=BiasSpec( + power_bias_decode=bias_decode, + power_bias_prefill=bias_prefill, + ), + ), + load=LoadSpec(profile="constant", tokens_per_sec=1000.0), + events=[], + assertions=[], + ) + + +def _make_overlay( + bias_decode: float = 1.0, + bias_prefill: float = 1.0, + seed: int = 42, + power_noise_sigma: float = 0.0, +): + """Build a SyntheticPowerOverlay matching the runtime injection point.""" + from dynamo.planner.tests.testbed.replay.synthetic_power_overlay import ( + SyntheticPowerOverlay, + ) + from dynamo.planner.tests.testbed.scenarios import NoiseModel, NoiseSpec, SystemSpec + + scenario = _make_scenario(bias_decode=bias_decode, bias_prefill=bias_prefill) + # Override noise if the test wants non-zero noise. + scenario.overlay.noise = NoiseSpec( + power_per_gpu=NoiseModel(model="gaussian", sigma=power_noise_sigma), + ) + system_spec = SystemSpec.load("h200_sxm") + rng = random.Random(seed) + return SyntheticPowerOverlay( + overlay_spec=scenario.overlay, + system_spec=system_spec, + scenario=scenario, + rng=rng, + ) + + +def _observe(overlay, tick: int = 0) -> tuple[float, float]: + """Drive one ``observe()`` tick with a representative FPM snapshot pair + and return ``(power_w_prefill, power_w_decode)``.""" + from dynamo.planner.tests.testbed.fake_actuator import AppliedCaps + + fpm = [ + {"component": "prefill", "sum_prefill_tokens": 4096, "sum_decode_kv_tokens": 0}, + { + "component": "decode", + "sum_prefill_tokens": 0, + "sum_decode_kv_tokens": 100_000, + }, + ] + overlay.observe(fpm, AppliedCaps(cap_p=450, cap_d=360), tick) + obs = overlay.observation_at(tick) + assert obs is not None, "observation_at returned None after observe()" + return obs.power_w_prefill, obs.power_w_decode + + +# --------------------------------------------------------------------------- +# Tests +# --------------------------------------------------------------------------- + + +class TestOverlayDeterminism: + def test_same_seed_same_output(self): + o1 = _make_overlay(seed=99) + o2 = _make_overlay(seed=99) + p1, d1 = _observe(o1) + p2, d2 = _observe(o2) + assert math.isclose(p1, p2, rel_tol=1e-9) + assert math.isclose(d1, d2, rel_tol=1e-9) + + def test_different_seeds_differ(self): + """With nonzero noise, different seeds must diverge.""" + o_a = _make_overlay(seed=1, power_noise_sigma=0.05) + o_b = _make_overlay(seed=2, power_noise_sigma=0.05) + _p_a, d_a = _observe(o_a) + _p_b, d_b = _observe(o_b) + assert not math.isclose( + d_a, d_b, rel_tol=1e-4 + ), f"Different seeds produced identical decode power ({d_a} vs {d_b})" + + +class TestOverlayBiasPropagation: + def test_decode_bias_doubles_decode_power(self): + base = _make_overlay(bias_decode=1.0, bias_prefill=1.0, seed=0) + biased = _make_overlay(bias_decode=2.0, bias_prefill=1.0, seed=0) + _, d_base = _observe(base) + _, d_biased = _observe(biased) + ratio = d_biased / d_base + assert math.isclose( + ratio, 2.0, rel_tol=1e-3 + ), f"Expected decode power to double; got ratio={ratio:.4f}" + + def test_decode_bias_does_not_affect_prefill(self): + base = _make_overlay(bias_decode=1.0, bias_prefill=1.0, seed=0) + biased = _make_overlay(bias_decode=2.0, bias_prefill=1.0, seed=0) + p_base, _ = _observe(base) + p_biased, _ = _observe(biased) + assert math.isclose( + p_base, p_biased, rel_tol=1e-3 + ), "Decode bias should not affect prefill power" + + def test_prefill_bias_scales_prefill_power(self): + base = _make_overlay(bias_decode=1.0, bias_prefill=1.0, seed=0) + biased = _make_overlay(bias_decode=1.0, bias_prefill=1.5, seed=0) + p_base, _ = _observe(base) + p_biased, _ = _observe(biased) + ratio = p_biased / p_base + assert math.isclose( + ratio, 1.5, rel_tol=1e-3 + ), f"Expected prefill power to scale by 1.5; got ratio={ratio:.4f}" + + +class TestOverlayZeroBias: + def test_no_bias_within_reasonable_envelope(self): + overlay = _make_overlay(bias_decode=1.0, bias_prefill=1.0, seed=0) + p_w, d_w = _observe(overlay) + assert 0 < p_w < 1000, f"Prefill power {p_w}W out of envelope" + assert 0 < d_w < 1000, f"Decode power {d_w}W out of envelope" diff --git a/components/src/dynamo/planner/tests/testbed/tests/test_scenarios_loadable.py b/components/src/dynamo/planner/tests/testbed/tests/test_scenarios_loadable.py new file mode 100644 index 000000000000..3e77304da02a --- /dev/null +++ b/components/src/dynamo/planner/tests/testbed/tests/test_scenarios_loadable.py @@ -0,0 +1,82 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Scenario YAML loadable gate. + +Every YAML in scenarios/ must: + 1. Parse without error (incl. ``extends:`` resolution). + 2. Pass Pydantic validation on every event and assertion. + 3. Reference only fields that exist on ``TickSnapshot``. + 4. Use only valid ``ref:`` prefixes (planner / counters / fleet / overlay). + 5. Have ``class: alpha`` or ``class: gamma``. + +The heavy lifting lives in ``scenarios.py`` (single source of truth); this +file just iterates and reports. +""" +from __future__ import annotations + +from pathlib import Path + +import pytest + +pytestmark = [ + pytest.mark.gpu_0, + pytest.mark.pre_merge, + pytest.mark.unit, + pytest.mark.planner, +] + +_SCENARIOS_DIR = Path(__file__).parent.parent / "scenarios" + + +def _iter_scenario_yamls() -> list[Path]: + """Return every non-base scenario YAML in sorted order.""" + return sorted( + p for p in _SCENARIOS_DIR.glob("*.yaml") if not p.stem.startswith("_") + ) + + +# --------------------------------------------------------------------------- +# Collect load-time errors at module import so failures surface during +# pytest collection rather than only when the test functions run. +# --------------------------------------------------------------------------- +_LOAD_ERRORS: list[str] = [] + +try: + from dynamo.planner.tests.testbed.scenarios import load_scenario + + for _yaml_path in _iter_scenario_yamls(): + try: + load_scenario(_yaml_path) + except Exception as _exc: + # Truncate huge ValidationError stacks; the first line names the + # offending field which is what authors need. + _LOAD_ERRORS.append(f" {_yaml_path.stem}: {str(_exc).splitlines()[0]}") +except ImportError as _e: # pragma: no cover — only when packaging is broken + _LOAD_ERRORS.append(f" Could not import scenarios module: {_e}") + + +def test_all_scenario_yamls_loadable() -> None: + """Every YAML in scenarios/ parses successfully with valid field/ref names.""" + if _LOAD_ERRORS: + msg = "\n".join(_LOAD_ERRORS) + pytest.fail( + f"{len(_LOAD_ERRORS)} scenario YAML error(s) detected at collection:\n{msg}" + ) + + +@pytest.mark.parametrize( + "scenario_path", + _iter_scenario_yamls(), + ids=[p.stem for p in _iter_scenario_yamls()], +) +def test_scenario_yaml_parses(scenario_path: Path) -> None: + """Each individual YAML parses, validates, and has a valid class label.""" + from dynamo.planner.tests.testbed.scenarios import load_scenario + + scenario = load_scenario(scenario_path) + assert scenario.name, f"{scenario_path.stem} has no name" + assert scenario.class_name in ( + "alpha", + "gamma", + ), f"{scenario_path.stem} has invalid class={scenario.class_name!r}" diff --git a/components/src/dynamo/planner/tests/testbed/tests/test_self_consistency.py b/components/src/dynamo/planner/tests/testbed/tests/test_self_consistency.py new file mode 100644 index 000000000000..38dccdf6d5d8 --- /dev/null +++ b/components/src/dynamo/planner/tests/testbed/tests/test_self_consistency.py @@ -0,0 +1,162 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Self-consistency tests for the testbed. + +Guards: + 1. test_alpha_no_bias: run A1 with power_bias_decode=1.0; c_power_d stays in [0.95, 1.05] + 2. test_gamma_no_bias: run G1 with power_bias_decode=1.0; c_power_d stays in [0.90, 1.10] + (skipped if dynamo.llm / mocker not available) + 3. test_alpha_gamma_agree_on_decode_drift: A1 and G1 must converge in the same + direction (both >1.0) within 20% magnitude +""" +from __future__ import annotations + +import statistics +from pathlib import Path +from typing import TYPE_CHECKING + +import pytest + +if TYPE_CHECKING: + from dynamo.planner.tests.testbed.recorder import TickHistory + +pytestmark = [ + pytest.mark.gpu_0, + pytest.mark.pre_merge, + pytest.mark.integration, + pytest.mark.planner, +] + +_SCENARIOS_DIR = Path(__file__).parent.parent / "scenarios" + + +def _run_scenario_with_bias( + scenario_name: str, power_bias_decode: float = 1.0 +) -> "TickHistory": + """Load a scenario, override decode bias to the given value, and run it.""" + from dynamo.planner.tests.testbed.runner import ScenarioRunner + from dynamo.planner.tests.testbed.scenarios import load_scenario + + yaml_path = _SCENARIOS_DIR / f"{scenario_name}.yaml" + scenario = load_scenario(yaml_path) + + # Override the decode power bias. Bias lives on FleetSpec.bias for α-class + # and OverlaySpec.bias for γ-class — NoiseSpec has no bias fields. + if scenario.fleet is not None and scenario.fleet.bias is not None: + scenario.fleet.bias.power_bias_decode = power_bias_decode + if scenario.overlay is not None and scenario.overlay.bias is not None: + scenario.overlay.bias.power_bias_decode = power_bias_decode + + runner = ScenarioRunner(scenario) + return runner.run() + + +def _median_c_power_d(history: "TickHistory") -> float: + values = [s.c_power_d for s in history.snapshots if s.c_power_d is not None] + if not values: + pytest.skip("No c_power_d values recorded") + return statistics.median(values) + + +def _late_avg_c_power_d(history: "TickHistory", last_n: int = 10) -> float: + values = [ + s.c_power_d for s in history.snapshots[-last_n:] if s.c_power_d is not None + ] + if not values: + pytest.skip("No c_power_d values in last ticks") + return sum(values) / len(values) + + +# --------------------------------------------------------------------------- +# α self-consistency +# --------------------------------------------------------------------------- +@pytest.mark.testbed +def test_alpha_no_bias() -> None: + """A1 with bias=1.0: c_power_d converges to [0.95, 1.05] (AIC-echo case).""" + history = _run_scenario_with_bias( + "A1_power_under_estimate_decode", power_bias_decode=1.0 + ) + avg = _late_avg_c_power_d(history) + assert ( + 0.90 <= avg <= 1.10 + ), f"α no-bias: expected c_power_d in [0.90, 1.10]; got {avg:.4f}" + + +# --------------------------------------------------------------------------- +# γ self-consistency +# --------------------------------------------------------------------------- +@pytest.mark.testbed +@pytest.mark.gamma +def test_gamma_no_bias() -> None: + """G1 with bias=1.0: c_power_d stays in [0.90, 1.10].""" + pytest.importorskip( + "dynamo.llm", + reason="γ self-consistency requires dynamo.llm (mocker) package", + ) + history = _run_scenario_with_bias( + "G1_realistic_decode_drift", power_bias_decode=1.0 + ) + avg = _late_avg_c_power_d(history) + assert ( + 0.90 <= avg <= 1.10 + ), f"γ no-bias: expected c_power_d in [0.90, 1.10]; got {avg:.4f}" + + +# --------------------------------------------------------------------------- +# α–γ cross-validation +# --------------------------------------------------------------------------- +@pytest.mark.testbed +@pytest.mark.gamma +def test_alpha_gamma_agree_on_decode_drift() -> None: + """A1 and G1 with bias=1.30 must agree on direction and magnitude (±20%). + + Both should converge to c_power_d > 1.0 (positive drift direction). + Magnitudes must be within 20% of each other, accounting for γ's broader + noise envelope. + """ + pytest.importorskip( + "dynamo.llm", + reason="α–γ cross-validation requires dynamo.llm (mocker) package", + ) + # Skip when the installed bridge only exposes the older create_disagg API. + # That API requires a trace file; the testbed falls back to a placeholder + # with near-zero load that cannot drive AIC power-correction drift, making + # this cross-validation meaningless. + try: + from dynamo.llm import PlannerReplayBridge # type: ignore[import] + + if not hasattr(PlannerReplayBridge, "from_synthetic_disagg"): + pytest.skip( + "PlannerReplayBridge.from_synthetic_disagg not available; " + "older create_disagg API with placeholder trace cannot drive " + "AIC c_power_d drift for α–γ cross-validation" + ) + except ImportError: + pass # already handled by importorskip above + bias = 1.30 + + alpha_history = _run_scenario_with_bias( + "A1_power_under_estimate_decode", power_bias_decode=bias + ) + gamma_history = _run_scenario_with_bias( + "G1_realistic_decode_drift", power_bias_decode=bias + ) + + alpha_avg = _late_avg_c_power_d(alpha_history) + gamma_avg = _late_avg_c_power_d(gamma_history) + + # Both must indicate positive drift (> 1.0 means underestimate corrected upward) + assert ( + alpha_avg > 1.0 + ), f"α A1: expected c_power_d > 1.0 with bias=1.30; got {alpha_avg:.4f}" + assert ( + gamma_avg > 1.0 + ), f"γ G1: expected c_power_d > 1.0 with bias=1.30; got {gamma_avg:.4f}" + + # Magnitude must agree within 20% + ratio = abs(alpha_avg - gamma_avg) / max(alpha_avg, gamma_avg) + assert ratio <= 0.20, ( + f"α–γ magnitude disagreement: α={alpha_avg:.4f}, γ={gamma_avg:.4f}, " + f"relative difference={ratio:.2%} (must be ≤20%)" + ) diff --git a/components/src/dynamo/planner/tests/testbed/traces/README.md b/components/src/dynamo/planner/tests/testbed/traces/README.md new file mode 100644 index 000000000000..9e53c2455eeb --- /dev/null +++ b/components/src/dynamo/planner/tests/testbed/traces/README.md @@ -0,0 +1,82 @@ +# Testbed Traces + +This directory holds `dynamo-mocker` traces for the γ-class (replay-based) +scenarios (`G1`, `G2`, `G3`). + +## Format — Mooncake JSONL (what `PlannerReplayBridge` actually consumes) + +Traces are `.jsonl` files in the **Mooncake request-trace format** that the +mocker scheduler consumes via `PlannerReplayBridge.{create_disagg, +from_trace_file_disagg}`. Every line is one *request* to be issued at the +specified simulated time: + +```json +{"timestamp": 0, "input_length": 512, "output_length": 50, "hash_ids": [0]} +{"timestamp": 12000, "input_length": 512, "output_length": 50, "hash_ids": [1]} +{"timestamp": 24000, "input_length": 512, "output_length": 50, "hash_ids": [2]} +``` + +Required fields per line: + +| Field | Type | Meaning | +|-------|------|---------| +| `timestamp` | int | Wall-clock milliseconds at which the request arrives (relative to trace start) | +| `input_length` | int | Prompt length in tokens | +| `output_length` | int | Decode length in tokens | +| `hash_ids` | list[int] | KV-block hash IDs for prefix-cache routing. One per `trace_block_size` block of input. | + +The bridge's `arrival_speedup_ratio` scales `timestamp` (e.g., 2.0 = arrivals +happen at half the wall-clock spacing). The engine's own `speedup_ratio` (set +in `MockEngineArgs`) scales the simulated processing time independently — +see Appendix D.5 of `powerplanner-testbed-design.md` for why this matters +for short test runs. + +> **Historical note**: an older internal harness used a different per-line +> format (FPM snapshots: `ts_ns`, `prefill_workers`, `decode_workers`). That +> format is **not** consumed by `PlannerReplayBridge` and produces +> `trace line N is missing hash_ids` if pointed at the current bridge. The +> placeholder bundled here has been regenerated in Mooncake format — +> see Appendix D.4. + +## Provided Traces + +| File | Description | Requests | Duration | +|------|-------------|---------:|---------:| +| `placeholder_h200_disagg_1rps.jsonl` | Synthetic placeholder for the older `create_disagg` bridge fallback. 300 small requests (`isl=512`, `osl=50`), one unique hash per request, staggered every 12 s. Used when a scenario specifies `synthetic_workload: true` but the installed bridge has no `from_synthetic_disagg` constructor. | 300 | 3588 s | + +> **Note**: Real captured traces are not committed to this repository due to +> size. Generate them with `dynamo-mocker` against the `h200-disagg` +> deployment config and place them here. The `ScenarioSpec.mocker.trace_file` +> field accepts an absolute path or a path relative to repo root. + +## Why the placeholder is small + sparse + +The placeholder is sized to keep the γ-class suite under 10 seconds of +wall-time, not to drive realistic load. With 300 small requests over +3600 simulated seconds at 1-worker capacity, fleet utilization is ≈ 0.003 — +enough to exercise the bridge's tick/scaling loop and the planner's +`reconcile_fpm_worker_count` path, but **not** enough to drive AIC EMA +correction. That's why `test_alpha_gamma_agree_on_decode_drift` auto-skips +when only the older bridge API is available (Appendix D.7). + +When the newer `from_synthetic_disagg` API is built into the dev image, +γ-class will drive proportional load through the synthetic-workload generator +and this placeholder will only be used in true `from_trace_file_disagg` +scenarios. + +## Generating Traces + +```bash +dynamo-mocker \ + --config examples/deployments/powerplanner/h200_disagg.yaml \ + --duration 120s \ + --dump-trace traces/my_trace.jsonl +``` + +Then reference the trace in your scenario: + +```yaml +mocker: + trace_file: components/src/dynamo/planner/tests/testbed/traces/my_trace.jsonl + synthetic_workload: false +``` diff --git a/components/src/dynamo/planner/tests/testbed/traces/placeholder_h200_disagg_1rps.jsonl b/components/src/dynamo/planner/tests/testbed/traces/placeholder_h200_disagg_1rps.jsonl new file mode 100644 index 000000000000..60f43e073cc1 --- /dev/null +++ b/components/src/dynamo/planner/tests/testbed/traces/placeholder_h200_disagg_1rps.jsonl @@ -0,0 +1,300 @@ +{"timestamp": 0, "input_length": 512, "output_length": 50, "hash_ids": [0]} +{"timestamp": 12000, "input_length": 512, "output_length": 50, "hash_ids": [1]} +{"timestamp": 24000, "input_length": 512, "output_length": 50, "hash_ids": [2]} +{"timestamp": 36000, "input_length": 512, "output_length": 50, "hash_ids": [3]} +{"timestamp": 48000, "input_length": 512, "output_length": 50, "hash_ids": [4]} +{"timestamp": 60000, "input_length": 512, "output_length": 50, "hash_ids": [5]} +{"timestamp": 72000, "input_length": 512, "output_length": 50, "hash_ids": [6]} +{"timestamp": 84000, "input_length": 512, "output_length": 50, "hash_ids": [7]} +{"timestamp": 96000, "input_length": 512, "output_length": 50, "hash_ids": [8]} +{"timestamp": 108000, "input_length": 512, "output_length": 50, "hash_ids": [9]} +{"timestamp": 120000, "input_length": 512, "output_length": 50, "hash_ids": [10]} +{"timestamp": 132000, "input_length": 512, "output_length": 50, "hash_ids": [11]} +{"timestamp": 144000, "input_length": 512, "output_length": 50, "hash_ids": [12]} +{"timestamp": 156000, "input_length": 512, "output_length": 50, "hash_ids": [13]} +{"timestamp": 168000, "input_length": 512, "output_length": 50, "hash_ids": [14]} +{"timestamp": 180000, "input_length": 512, "output_length": 50, "hash_ids": [15]} +{"timestamp": 192000, "input_length": 512, "output_length": 50, "hash_ids": [16]} +{"timestamp": 204000, "input_length": 512, "output_length": 50, "hash_ids": [17]} +{"timestamp": 216000, "input_length": 512, "output_length": 50, "hash_ids": [18]} +{"timestamp": 228000, "input_length": 512, "output_length": 50, "hash_ids": [19]} +{"timestamp": 240000, "input_length": 512, "output_length": 50, "hash_ids": [20]} +{"timestamp": 252000, "input_length": 512, "output_length": 50, "hash_ids": [21]} +{"timestamp": 264000, "input_length": 512, "output_length": 50, "hash_ids": [22]} +{"timestamp": 276000, "input_length": 512, "output_length": 50, "hash_ids": [23]} +{"timestamp": 288000, "input_length": 512, "output_length": 50, "hash_ids": [24]} +{"timestamp": 300000, "input_length": 512, "output_length": 50, "hash_ids": [25]} +{"timestamp": 312000, "input_length": 512, "output_length": 50, "hash_ids": [26]} +{"timestamp": 324000, "input_length": 512, "output_length": 50, "hash_ids": [27]} +{"timestamp": 336000, "input_length": 512, "output_length": 50, "hash_ids": [28]} +{"timestamp": 348000, "input_length": 512, "output_length": 50, "hash_ids": [29]} +{"timestamp": 360000, "input_length": 512, "output_length": 50, "hash_ids": [30]} +{"timestamp": 372000, "input_length": 512, "output_length": 50, "hash_ids": [31]} +{"timestamp": 384000, "input_length": 512, "output_length": 50, "hash_ids": [32]} +{"timestamp": 396000, "input_length": 512, "output_length": 50, "hash_ids": [33]} +{"timestamp": 408000, "input_length": 512, "output_length": 50, "hash_ids": [34]} +{"timestamp": 420000, "input_length": 512, "output_length": 50, "hash_ids": [35]} +{"timestamp": 432000, "input_length": 512, "output_length": 50, "hash_ids": [36]} +{"timestamp": 444000, "input_length": 512, "output_length": 50, "hash_ids": [37]} +{"timestamp": 456000, "input_length": 512, "output_length": 50, "hash_ids": [38]} +{"timestamp": 468000, "input_length": 512, "output_length": 50, "hash_ids": [39]} +{"timestamp": 480000, "input_length": 512, "output_length": 50, "hash_ids": [40]} +{"timestamp": 492000, "input_length": 512, "output_length": 50, "hash_ids": [41]} +{"timestamp": 504000, "input_length": 512, "output_length": 50, "hash_ids": [42]} +{"timestamp": 516000, "input_length": 512, "output_length": 50, "hash_ids": [43]} +{"timestamp": 528000, "input_length": 512, "output_length": 50, "hash_ids": [44]} +{"timestamp": 540000, "input_length": 512, "output_length": 50, "hash_ids": [45]} +{"timestamp": 552000, "input_length": 512, "output_length": 50, "hash_ids": [46]} +{"timestamp": 564000, "input_length": 512, "output_length": 50, "hash_ids": [47]} +{"timestamp": 576000, "input_length": 512, "output_length": 50, "hash_ids": [48]} +{"timestamp": 588000, "input_length": 512, "output_length": 50, "hash_ids": [49]} +{"timestamp": 600000, "input_length": 512, "output_length": 50, "hash_ids": [50]} +{"timestamp": 612000, "input_length": 512, "output_length": 50, "hash_ids": [51]} +{"timestamp": 624000, "input_length": 512, "output_length": 50, "hash_ids": [52]} +{"timestamp": 636000, "input_length": 512, "output_length": 50, "hash_ids": [53]} +{"timestamp": 648000, "input_length": 512, "output_length": 50, "hash_ids": [54]} +{"timestamp": 660000, "input_length": 512, "output_length": 50, "hash_ids": [55]} +{"timestamp": 672000, "input_length": 512, "output_length": 50, "hash_ids": [56]} +{"timestamp": 684000, "input_length": 512, "output_length": 50, "hash_ids": [57]} +{"timestamp": 696000, "input_length": 512, "output_length": 50, "hash_ids": [58]} +{"timestamp": 708000, "input_length": 512, "output_length": 50, "hash_ids": [59]} +{"timestamp": 720000, "input_length": 512, "output_length": 50, "hash_ids": [60]} +{"timestamp": 732000, "input_length": 512, "output_length": 50, "hash_ids": [61]} +{"timestamp": 744000, "input_length": 512, "output_length": 50, "hash_ids": [62]} +{"timestamp": 756000, "input_length": 512, "output_length": 50, "hash_ids": [63]} +{"timestamp": 768000, "input_length": 512, "output_length": 50, "hash_ids": [64]} +{"timestamp": 780000, "input_length": 512, "output_length": 50, "hash_ids": [65]} +{"timestamp": 792000, "input_length": 512, "output_length": 50, "hash_ids": [66]} +{"timestamp": 804000, "input_length": 512, "output_length": 50, "hash_ids": [67]} +{"timestamp": 816000, "input_length": 512, "output_length": 50, "hash_ids": [68]} +{"timestamp": 828000, "input_length": 512, "output_length": 50, "hash_ids": [69]} +{"timestamp": 840000, "input_length": 512, "output_length": 50, "hash_ids": [70]} +{"timestamp": 852000, "input_length": 512, "output_length": 50, "hash_ids": [71]} +{"timestamp": 864000, "input_length": 512, "output_length": 50, "hash_ids": [72]} +{"timestamp": 876000, "input_length": 512, "output_length": 50, "hash_ids": [73]} +{"timestamp": 888000, "input_length": 512, "output_length": 50, "hash_ids": [74]} +{"timestamp": 900000, "input_length": 512, "output_length": 50, "hash_ids": [75]} +{"timestamp": 912000, "input_length": 512, "output_length": 50, "hash_ids": [76]} +{"timestamp": 924000, "input_length": 512, "output_length": 50, "hash_ids": [77]} +{"timestamp": 936000, "input_length": 512, "output_length": 50, "hash_ids": [78]} +{"timestamp": 948000, "input_length": 512, "output_length": 50, "hash_ids": [79]} +{"timestamp": 960000, "input_length": 512, "output_length": 50, "hash_ids": [80]} +{"timestamp": 972000, "input_length": 512, "output_length": 50, "hash_ids": [81]} +{"timestamp": 984000, "input_length": 512, "output_length": 50, "hash_ids": [82]} +{"timestamp": 996000, "input_length": 512, "output_length": 50, "hash_ids": [83]} +{"timestamp": 1008000, "input_length": 512, "output_length": 50, "hash_ids": [84]} +{"timestamp": 1020000, "input_length": 512, "output_length": 50, "hash_ids": [85]} +{"timestamp": 1032000, "input_length": 512, "output_length": 50, "hash_ids": [86]} +{"timestamp": 1044000, "input_length": 512, "output_length": 50, "hash_ids": [87]} +{"timestamp": 1056000, "input_length": 512, "output_length": 50, "hash_ids": [88]} +{"timestamp": 1068000, "input_length": 512, "output_length": 50, "hash_ids": [89]} +{"timestamp": 1080000, "input_length": 512, "output_length": 50, "hash_ids": [90]} +{"timestamp": 1092000, "input_length": 512, "output_length": 50, "hash_ids": [91]} +{"timestamp": 1104000, "input_length": 512, "output_length": 50, "hash_ids": [92]} +{"timestamp": 1116000, "input_length": 512, "output_length": 50, "hash_ids": [93]} +{"timestamp": 1128000, "input_length": 512, "output_length": 50, "hash_ids": [94]} +{"timestamp": 1140000, "input_length": 512, "output_length": 50, "hash_ids": [95]} +{"timestamp": 1152000, "input_length": 512, "output_length": 50, "hash_ids": [96]} +{"timestamp": 1164000, "input_length": 512, "output_length": 50, "hash_ids": [97]} +{"timestamp": 1176000, "input_length": 512, "output_length": 50, "hash_ids": [98]} +{"timestamp": 1188000, "input_length": 512, "output_length": 50, "hash_ids": [99]} +{"timestamp": 1200000, "input_length": 512, "output_length": 50, "hash_ids": [100]} +{"timestamp": 1212000, "input_length": 512, "output_length": 50, "hash_ids": [101]} +{"timestamp": 1224000, "input_length": 512, "output_length": 50, "hash_ids": [102]} +{"timestamp": 1236000, "input_length": 512, "output_length": 50, "hash_ids": [103]} +{"timestamp": 1248000, "input_length": 512, "output_length": 50, "hash_ids": [104]} +{"timestamp": 1260000, "input_length": 512, "output_length": 50, "hash_ids": [105]} +{"timestamp": 1272000, "input_length": 512, "output_length": 50, "hash_ids": [106]} +{"timestamp": 1284000, "input_length": 512, "output_length": 50, "hash_ids": [107]} +{"timestamp": 1296000, "input_length": 512, "output_length": 50, "hash_ids": [108]} +{"timestamp": 1308000, "input_length": 512, "output_length": 50, "hash_ids": [109]} +{"timestamp": 1320000, "input_length": 512, "output_length": 50, "hash_ids": [110]} +{"timestamp": 1332000, "input_length": 512, "output_length": 50, "hash_ids": [111]} +{"timestamp": 1344000, "input_length": 512, "output_length": 50, "hash_ids": [112]} +{"timestamp": 1356000, "input_length": 512, "output_length": 50, "hash_ids": [113]} +{"timestamp": 1368000, "input_length": 512, "output_length": 50, "hash_ids": [114]} +{"timestamp": 1380000, "input_length": 512, "output_length": 50, "hash_ids": [115]} +{"timestamp": 1392000, "input_length": 512, "output_length": 50, "hash_ids": [116]} +{"timestamp": 1404000, "input_length": 512, "output_length": 50, "hash_ids": [117]} +{"timestamp": 1416000, "input_length": 512, "output_length": 50, "hash_ids": [118]} +{"timestamp": 1428000, "input_length": 512, "output_length": 50, "hash_ids": [119]} +{"timestamp": 1440000, "input_length": 512, "output_length": 50, "hash_ids": [120]} +{"timestamp": 1452000, "input_length": 512, "output_length": 50, "hash_ids": [121]} +{"timestamp": 1464000, "input_length": 512, "output_length": 50, "hash_ids": [122]} +{"timestamp": 1476000, "input_length": 512, "output_length": 50, "hash_ids": [123]} +{"timestamp": 1488000, "input_length": 512, "output_length": 50, "hash_ids": [124]} +{"timestamp": 1500000, "input_length": 512, "output_length": 50, "hash_ids": [125]} +{"timestamp": 1512000, "input_length": 512, "output_length": 50, "hash_ids": [126]} +{"timestamp": 1524000, "input_length": 512, "output_length": 50, "hash_ids": [127]} +{"timestamp": 1536000, "input_length": 512, "output_length": 50, "hash_ids": [128]} +{"timestamp": 1548000, "input_length": 512, "output_length": 50, "hash_ids": [129]} +{"timestamp": 1560000, "input_length": 512, "output_length": 50, "hash_ids": [130]} +{"timestamp": 1572000, "input_length": 512, "output_length": 50, "hash_ids": [131]} +{"timestamp": 1584000, "input_length": 512, "output_length": 50, "hash_ids": [132]} +{"timestamp": 1596000, "input_length": 512, "output_length": 50, "hash_ids": [133]} +{"timestamp": 1608000, "input_length": 512, "output_length": 50, "hash_ids": [134]} +{"timestamp": 1620000, "input_length": 512, "output_length": 50, "hash_ids": [135]} +{"timestamp": 1632000, "input_length": 512, "output_length": 50, "hash_ids": [136]} +{"timestamp": 1644000, "input_length": 512, "output_length": 50, "hash_ids": [137]} +{"timestamp": 1656000, "input_length": 512, "output_length": 50, "hash_ids": [138]} +{"timestamp": 1668000, "input_length": 512, "output_length": 50, "hash_ids": [139]} +{"timestamp": 1680000, "input_length": 512, "output_length": 50, "hash_ids": [140]} +{"timestamp": 1692000, "input_length": 512, "output_length": 50, "hash_ids": [141]} +{"timestamp": 1704000, "input_length": 512, "output_length": 50, "hash_ids": [142]} +{"timestamp": 1716000, "input_length": 512, "output_length": 50, "hash_ids": [143]} +{"timestamp": 1728000, "input_length": 512, "output_length": 50, "hash_ids": [144]} +{"timestamp": 1740000, "input_length": 512, "output_length": 50, "hash_ids": [145]} +{"timestamp": 1752000, "input_length": 512, "output_length": 50, "hash_ids": [146]} +{"timestamp": 1764000, "input_length": 512, "output_length": 50, "hash_ids": [147]} +{"timestamp": 1776000, "input_length": 512, "output_length": 50, "hash_ids": [148]} +{"timestamp": 1788000, "input_length": 512, "output_length": 50, "hash_ids": [149]} +{"timestamp": 1800000, "input_length": 512, "output_length": 50, "hash_ids": [150]} +{"timestamp": 1812000, "input_length": 512, "output_length": 50, "hash_ids": [151]} +{"timestamp": 1824000, "input_length": 512, "output_length": 50, "hash_ids": [152]} +{"timestamp": 1836000, "input_length": 512, "output_length": 50, "hash_ids": [153]} +{"timestamp": 1848000, "input_length": 512, "output_length": 50, "hash_ids": [154]} +{"timestamp": 1860000, "input_length": 512, "output_length": 50, "hash_ids": [155]} +{"timestamp": 1872000, "input_length": 512, "output_length": 50, "hash_ids": [156]} +{"timestamp": 1884000, "input_length": 512, "output_length": 50, "hash_ids": [157]} +{"timestamp": 1896000, "input_length": 512, "output_length": 50, "hash_ids": [158]} +{"timestamp": 1908000, "input_length": 512, "output_length": 50, "hash_ids": [159]} +{"timestamp": 1920000, "input_length": 512, "output_length": 50, "hash_ids": [160]} +{"timestamp": 1932000, "input_length": 512, "output_length": 50, "hash_ids": [161]} +{"timestamp": 1944000, "input_length": 512, "output_length": 50, "hash_ids": [162]} +{"timestamp": 1956000, "input_length": 512, "output_length": 50, "hash_ids": [163]} +{"timestamp": 1968000, "input_length": 512, "output_length": 50, "hash_ids": [164]} +{"timestamp": 1980000, "input_length": 512, "output_length": 50, "hash_ids": [165]} +{"timestamp": 1992000, "input_length": 512, "output_length": 50, "hash_ids": [166]} +{"timestamp": 2004000, "input_length": 512, "output_length": 50, "hash_ids": [167]} +{"timestamp": 2016000, "input_length": 512, "output_length": 50, "hash_ids": [168]} +{"timestamp": 2028000, "input_length": 512, "output_length": 50, "hash_ids": [169]} +{"timestamp": 2040000, "input_length": 512, "output_length": 50, "hash_ids": [170]} +{"timestamp": 2052000, "input_length": 512, "output_length": 50, "hash_ids": [171]} +{"timestamp": 2064000, "input_length": 512, "output_length": 50, "hash_ids": [172]} +{"timestamp": 2076000, "input_length": 512, "output_length": 50, "hash_ids": [173]} +{"timestamp": 2088000, "input_length": 512, "output_length": 50, "hash_ids": [174]} +{"timestamp": 2100000, "input_length": 512, "output_length": 50, "hash_ids": [175]} +{"timestamp": 2112000, "input_length": 512, "output_length": 50, "hash_ids": [176]} +{"timestamp": 2124000, "input_length": 512, "output_length": 50, "hash_ids": [177]} +{"timestamp": 2136000, "input_length": 512, "output_length": 50, "hash_ids": [178]} +{"timestamp": 2148000, "input_length": 512, "output_length": 50, "hash_ids": [179]} +{"timestamp": 2160000, "input_length": 512, "output_length": 50, "hash_ids": [180]} +{"timestamp": 2172000, "input_length": 512, "output_length": 50, "hash_ids": [181]} +{"timestamp": 2184000, "input_length": 512, "output_length": 50, "hash_ids": [182]} +{"timestamp": 2196000, "input_length": 512, "output_length": 50, "hash_ids": [183]} +{"timestamp": 2208000, "input_length": 512, "output_length": 50, "hash_ids": [184]} +{"timestamp": 2220000, "input_length": 512, "output_length": 50, "hash_ids": [185]} +{"timestamp": 2232000, "input_length": 512, "output_length": 50, "hash_ids": [186]} +{"timestamp": 2244000, "input_length": 512, "output_length": 50, "hash_ids": [187]} +{"timestamp": 2256000, "input_length": 512, "output_length": 50, "hash_ids": [188]} +{"timestamp": 2268000, "input_length": 512, "output_length": 50, "hash_ids": [189]} +{"timestamp": 2280000, "input_length": 512, "output_length": 50, "hash_ids": [190]} +{"timestamp": 2292000, "input_length": 512, "output_length": 50, "hash_ids": [191]} +{"timestamp": 2304000, "input_length": 512, "output_length": 50, "hash_ids": [192]} +{"timestamp": 2316000, "input_length": 512, "output_length": 50, "hash_ids": [193]} +{"timestamp": 2328000, "input_length": 512, "output_length": 50, "hash_ids": [194]} +{"timestamp": 2340000, "input_length": 512, "output_length": 50, "hash_ids": [195]} +{"timestamp": 2352000, "input_length": 512, "output_length": 50, "hash_ids": [196]} +{"timestamp": 2364000, "input_length": 512, "output_length": 50, "hash_ids": [197]} +{"timestamp": 2376000, "input_length": 512, "output_length": 50, "hash_ids": [198]} +{"timestamp": 2388000, "input_length": 512, "output_length": 50, "hash_ids": [199]} +{"timestamp": 2400000, "input_length": 512, "output_length": 50, "hash_ids": [200]} +{"timestamp": 2412000, "input_length": 512, "output_length": 50, "hash_ids": [201]} +{"timestamp": 2424000, "input_length": 512, "output_length": 50, "hash_ids": [202]} +{"timestamp": 2436000, "input_length": 512, "output_length": 50, "hash_ids": [203]} +{"timestamp": 2448000, "input_length": 512, "output_length": 50, "hash_ids": [204]} +{"timestamp": 2460000, "input_length": 512, "output_length": 50, "hash_ids": [205]} +{"timestamp": 2472000, "input_length": 512, "output_length": 50, "hash_ids": [206]} +{"timestamp": 2484000, "input_length": 512, "output_length": 50, "hash_ids": [207]} +{"timestamp": 2496000, "input_length": 512, "output_length": 50, "hash_ids": [208]} +{"timestamp": 2508000, "input_length": 512, "output_length": 50, "hash_ids": [209]} +{"timestamp": 2520000, "input_length": 512, "output_length": 50, "hash_ids": [210]} +{"timestamp": 2532000, "input_length": 512, "output_length": 50, "hash_ids": [211]} +{"timestamp": 2544000, "input_length": 512, "output_length": 50, "hash_ids": [212]} +{"timestamp": 2556000, "input_length": 512, "output_length": 50, "hash_ids": [213]} +{"timestamp": 2568000, "input_length": 512, "output_length": 50, "hash_ids": [214]} +{"timestamp": 2580000, "input_length": 512, "output_length": 50, "hash_ids": [215]} +{"timestamp": 2592000, "input_length": 512, "output_length": 50, "hash_ids": [216]} +{"timestamp": 2604000, "input_length": 512, "output_length": 50, "hash_ids": [217]} +{"timestamp": 2616000, "input_length": 512, "output_length": 50, "hash_ids": [218]} +{"timestamp": 2628000, "input_length": 512, "output_length": 50, "hash_ids": [219]} +{"timestamp": 2640000, "input_length": 512, "output_length": 50, "hash_ids": [220]} +{"timestamp": 2652000, "input_length": 512, "output_length": 50, "hash_ids": [221]} +{"timestamp": 2664000, "input_length": 512, "output_length": 50, "hash_ids": [222]} +{"timestamp": 2676000, "input_length": 512, "output_length": 50, "hash_ids": [223]} +{"timestamp": 2688000, "input_length": 512, "output_length": 50, "hash_ids": [224]} +{"timestamp": 2700000, "input_length": 512, "output_length": 50, "hash_ids": [225]} +{"timestamp": 2712000, "input_length": 512, "output_length": 50, "hash_ids": [226]} +{"timestamp": 2724000, "input_length": 512, "output_length": 50, "hash_ids": [227]} +{"timestamp": 2736000, "input_length": 512, "output_length": 50, "hash_ids": [228]} +{"timestamp": 2748000, "input_length": 512, "output_length": 50, "hash_ids": [229]} +{"timestamp": 2760000, "input_length": 512, "output_length": 50, "hash_ids": [230]} +{"timestamp": 2772000, "input_length": 512, "output_length": 50, "hash_ids": [231]} +{"timestamp": 2784000, "input_length": 512, "output_length": 50, "hash_ids": [232]} +{"timestamp": 2796000, "input_length": 512, "output_length": 50, "hash_ids": [233]} +{"timestamp": 2808000, "input_length": 512, "output_length": 50, "hash_ids": [234]} +{"timestamp": 2820000, "input_length": 512, "output_length": 50, "hash_ids": [235]} +{"timestamp": 2832000, "input_length": 512, "output_length": 50, "hash_ids": [236]} +{"timestamp": 2844000, "input_length": 512, "output_length": 50, "hash_ids": [237]} +{"timestamp": 2856000, "input_length": 512, "output_length": 50, "hash_ids": [238]} +{"timestamp": 2868000, "input_length": 512, "output_length": 50, "hash_ids": [239]} +{"timestamp": 2880000, "input_length": 512, "output_length": 50, "hash_ids": [240]} +{"timestamp": 2892000, "input_length": 512, "output_length": 50, "hash_ids": [241]} +{"timestamp": 2904000, "input_length": 512, "output_length": 50, "hash_ids": [242]} +{"timestamp": 2916000, "input_length": 512, "output_length": 50, "hash_ids": [243]} +{"timestamp": 2928000, "input_length": 512, "output_length": 50, "hash_ids": [244]} +{"timestamp": 2940000, "input_length": 512, "output_length": 50, "hash_ids": [245]} +{"timestamp": 2952000, "input_length": 512, "output_length": 50, "hash_ids": [246]} +{"timestamp": 2964000, "input_length": 512, "output_length": 50, "hash_ids": [247]} +{"timestamp": 2976000, "input_length": 512, "output_length": 50, "hash_ids": [248]} +{"timestamp": 2988000, "input_length": 512, "output_length": 50, "hash_ids": [249]} +{"timestamp": 3000000, "input_length": 512, "output_length": 50, "hash_ids": [250]} +{"timestamp": 3012000, "input_length": 512, "output_length": 50, "hash_ids": [251]} +{"timestamp": 3024000, "input_length": 512, "output_length": 50, "hash_ids": [252]} +{"timestamp": 3036000, "input_length": 512, "output_length": 50, "hash_ids": [253]} +{"timestamp": 3048000, "input_length": 512, "output_length": 50, "hash_ids": [254]} +{"timestamp": 3060000, "input_length": 512, "output_length": 50, "hash_ids": [255]} +{"timestamp": 3072000, "input_length": 512, "output_length": 50, "hash_ids": [256]} +{"timestamp": 3084000, "input_length": 512, "output_length": 50, "hash_ids": [257]} +{"timestamp": 3096000, "input_length": 512, "output_length": 50, "hash_ids": [258]} +{"timestamp": 3108000, "input_length": 512, "output_length": 50, "hash_ids": [259]} +{"timestamp": 3120000, "input_length": 512, "output_length": 50, "hash_ids": [260]} +{"timestamp": 3132000, "input_length": 512, "output_length": 50, "hash_ids": [261]} +{"timestamp": 3144000, "input_length": 512, "output_length": 50, "hash_ids": [262]} +{"timestamp": 3156000, "input_length": 512, "output_length": 50, "hash_ids": [263]} +{"timestamp": 3168000, "input_length": 512, "output_length": 50, "hash_ids": [264]} +{"timestamp": 3180000, "input_length": 512, "output_length": 50, "hash_ids": [265]} +{"timestamp": 3192000, "input_length": 512, "output_length": 50, "hash_ids": [266]} +{"timestamp": 3204000, "input_length": 512, "output_length": 50, "hash_ids": [267]} +{"timestamp": 3216000, "input_length": 512, "output_length": 50, "hash_ids": [268]} +{"timestamp": 3228000, "input_length": 512, "output_length": 50, "hash_ids": [269]} +{"timestamp": 3240000, "input_length": 512, "output_length": 50, "hash_ids": [270]} +{"timestamp": 3252000, "input_length": 512, "output_length": 50, "hash_ids": [271]} +{"timestamp": 3264000, "input_length": 512, "output_length": 50, "hash_ids": [272]} +{"timestamp": 3276000, "input_length": 512, "output_length": 50, "hash_ids": [273]} +{"timestamp": 3288000, "input_length": 512, "output_length": 50, "hash_ids": [274]} +{"timestamp": 3300000, "input_length": 512, "output_length": 50, "hash_ids": [275]} +{"timestamp": 3312000, "input_length": 512, "output_length": 50, "hash_ids": [276]} +{"timestamp": 3324000, "input_length": 512, "output_length": 50, "hash_ids": [277]} +{"timestamp": 3336000, "input_length": 512, "output_length": 50, "hash_ids": [278]} +{"timestamp": 3348000, "input_length": 512, "output_length": 50, "hash_ids": [279]} +{"timestamp": 3360000, "input_length": 512, "output_length": 50, "hash_ids": [280]} +{"timestamp": 3372000, "input_length": 512, "output_length": 50, "hash_ids": [281]} +{"timestamp": 3384000, "input_length": 512, "output_length": 50, "hash_ids": [282]} +{"timestamp": 3396000, "input_length": 512, "output_length": 50, "hash_ids": [283]} +{"timestamp": 3408000, "input_length": 512, "output_length": 50, "hash_ids": [284]} +{"timestamp": 3420000, "input_length": 512, "output_length": 50, "hash_ids": [285]} +{"timestamp": 3432000, "input_length": 512, "output_length": 50, "hash_ids": [286]} +{"timestamp": 3444000, "input_length": 512, "output_length": 50, "hash_ids": [287]} +{"timestamp": 3456000, "input_length": 512, "output_length": 50, "hash_ids": [288]} +{"timestamp": 3468000, "input_length": 512, "output_length": 50, "hash_ids": [289]} +{"timestamp": 3480000, "input_length": 512, "output_length": 50, "hash_ids": [290]} +{"timestamp": 3492000, "input_length": 512, "output_length": 50, "hash_ids": [291]} +{"timestamp": 3504000, "input_length": 512, "output_length": 50, "hash_ids": [292]} +{"timestamp": 3516000, "input_length": 512, "output_length": 50, "hash_ids": [293]} +{"timestamp": 3528000, "input_length": 512, "output_length": 50, "hash_ids": [294]} +{"timestamp": 3540000, "input_length": 512, "output_length": 50, "hash_ids": [295]} +{"timestamp": 3552000, "input_length": 512, "output_length": 50, "hash_ids": [296]} +{"timestamp": 3564000, "input_length": 512, "output_length": 50, "hash_ids": [297]} +{"timestamp": 3576000, "input_length": 512, "output_length": 50, "hash_ids": [298]} +{"timestamp": 3588000, "input_length": 512, "output_length": 50, "hash_ids": [299]} From 4815082abfa87d5e9fc109366e227acc22008280 Mon Sep 17 00:00:00 2001 From: Kai Ma Date: Tue, 19 May 2026 14:17:22 -0400 Subject: [PATCH 2/2] docs(testbed): soften cross-PR forward refs in testbed README The two ``[Appendix C.10]``/``[Appendix D.7]`` links in the testbed README pointed at ``docs/design-docs/powerplanner-testbed-design.md``, which is introduced by PR #9687 and does not yet exist on this branch. The Docs link check (lychee) has therefore failed on PR #9686 since the PR was opened on 2026-05-18 -- a cross-PR forward reference baked into the original PR #9369 split. Convert both occurrences from ``[text](relative-path)`` syntax to plain backticked text references. The information value (appendix numbers + target file) is preserved; lychee no longer treats them as candidate links to resolve. Once both PRs land on ``main`` the file resolves naturally and reviewers can grep for the path. No code or test change. Cascade-affected branches: this commit lives on pr4/testbed; pr5/docs-devenv will rebase onto the new tip. Signed-off-by: Kai Ma --- components/src/dynamo/planner/tests/testbed/README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/components/src/dynamo/planner/tests/testbed/README.md b/components/src/dynamo/planner/tests/testbed/README.md index 582279175e99..0f950a297a87 100644 --- a/components/src/dynamo/planner/tests/testbed/README.md +++ b/components/src/dynamo/planner/tests/testbed/README.md @@ -129,12 +129,12 @@ import time with a one-line reason. The marker is `@pytest.mark.real_aic`. 1. **No `dynamo._core` native binding** (e.g., fresh dev box without `maturin`): the runtime stub installs a no-op `dynamo._core` and `conftest.py`'s `pytest_collection_modifyitems` hook skips every `@pytest.mark.gamma` test. - See [Appendix C.10](../../../../../../docs/design-docs/powerplanner-testbed-design.md). + See Appendix C.10 in `docs/design-docs/powerplanner-testbed-design.md`. 2. **Older bridge API only** (`create_disagg` without `from_synthetic_disagg`): the α–γ cross-validation test (`test_alpha_gamma_agree_on_decode_drift`) skips because the placeholder trace fallback can't drive AIC drift. The - three γ scenarios + `test_gamma_no_bias` still run. See - [Appendix D.7](../../../../../../docs/design-docs/powerplanner-testbed-design.md). + three γ scenarios + `test_gamma_no_bias` still run. See Appendix D.7 in + `docs/design-docs/powerplanner-testbed-design.md`. Expected outcomes by environment: